Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TASK-6981 - Should not use same Solr timeout for reading and for indexing #93

Open
wants to merge 4 commits into
base: release-5.x.x
Choose a base branch
from
Open

TASK-6981 - Should not use same Solr timeout for reading and for indexing #93

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b49926c
datastore: add a method to SolrManager to create a Solr client, #TASK…
jtarraga Sep 27, 2024
444088d
datastore: use final for solrClient, #TASK-6981
jtarraga Sep 30, 2024
179f751
datastore: use final in SolrManager members, #TASK-6981
jtarraga Oct 2, 2024
51748f2
cicd: Fix calculating opencga branch in pull request approve workflow…
juanfeSanahuja Oct 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions .github/workflows/test-xetabase-workflow.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,18 @@ jobs:
path: opencga-enterprise
fetch-depth: "10"
- id: get_opencga_branch
name: Get OpenCGA branch from 'pom.xml' property
name: Get OpenCGA branch
run: |
pwd
ls -lrtha
ls -lrtha ./opencga-enterprise
chmod +x ./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh
opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh)
# If the task exists in the opencga repository, this is the branch to be tested
if [[ "${{ inputs.task }}" == TASK* ]]; then
if [ "$(git ls-remote "https://github.com/opencb/opencga.git" "${{ inputs.task }}" ] ; then
opencga_branch="${{ inputs.task }}";
return 0;
fi
else
chmod +x ./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh
opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh)
fi
echo "opencga_branch=${opencga_branch}"
echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT
- name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}'
Expand Down
114 changes: 48 additions & 66 deletions ...e/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,22 +23,26 @@
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.client.solrj.request.*;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreStatus;
import org.apache.solr.client.solrj.request.SolrPing;
import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.common.SolrException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

public class SolrManager {

private String host;
private String mode;
private SolrClient solrClient;
private final List<String> hosts;
private final String mode;
private final SolrClient solrClient;

private final Logger logger = LoggerFactory.getLogger(SolrManager.class);

Expand All @@ -65,49 +69,14 @@ public SolrManager(String host, String mode, int timeout) {
* @param timeout Read timeout
*/
public SolrManager(List<String> hosts, String mode, int timeout) {
hosts = hosts.stream().flatMap(s -> Arrays.stream(s.split(","))).collect(Collectors.toList());

this.host = String.join(",", hosts);
this.hosts = hosts.stream().flatMap(s -> Arrays.stream(s.split(","))).collect(Collectors.toList());
this.mode = mode;

if (hosts.get(0).startsWith("http")) {
if (hosts.size() == 1) {
// Single HTTP endpoint.
this.solrClient = new HttpSolrClient.Builder(host).build();
((HttpSolrClient) this.solrClient).setRequestWriter(new BinaryRequestWriter());
((HttpSolrClient) this.solrClient).setSoTimeout(timeout);
} else {
// Use a LoadBalancer if there are multiple http hosts
this.solrClient = new LBHttpSolrClient.Builder().withBaseSolrUrls(hosts.toArray(new String[0])).build();

((LBHttpSolrClient) this.solrClient).setRequestWriter(new BinaryRequestWriter());
((LBHttpSolrClient) this.solrClient).setSoTimeout(timeout);
}
} else {
// If the provided hosts are not http, assume zookeeper hosts like HOST:PORT
// This client will use Zookeeper to discover Solr endpoints for SolrCloud collections, and then use the
// LBHttpSolrClient to issue requests.
if (isCloud()) {
this.solrClient = new CloudSolrClient.Builder().withZkHost(hosts).build();

((CloudSolrClient) this.solrClient).setRequestWriter(new BinaryRequestWriter());
((CloudSolrClient) this.solrClient).setSoTimeout(timeout);
} else {
throw new IllegalArgumentException("Can not initialize SolrManager from Zookeeper host not in Cloud mode");
}
}
this.solrClient = newSolrClient(timeout);
}

public SolrManager(SolrClient solrClient, String host, String mode) {
this.solrClient = solrClient;
this.host = host;
this.mode = mode;
}

@Deprecated
public SolrManager(SolrClient solrClient, String host, String mode, int timeout) {
this.solrClient = solrClient;
this.host = host;
this.hosts = Collections.singletonList(host);
this.mode = mode;
}

Expand Down Expand Up @@ -188,7 +157,7 @@ public void create(String dbName, String configSet) throws SolrException {
*/
public void createCore(String coreName, String configSet) throws SolrException {
try {
logger.debug("Creating core: host={}, core={}, configSet={}", host, coreName, configSet);
logger.debug("Creating core: host={}, core={}, configSet={}", StringUtils.join(",", hosts), coreName, configSet);
CoreAdminRequest.Create request = new CoreAdminRequest.Create();
request.setCoreName(coreName);
request.setConfigSet(configSet);
Expand All @@ -210,7 +179,7 @@ public void createCore(String coreName, String configSet) throws SolrException {
*/
public void createCollection(String collectionName, String configSet) throws SolrException {
logger.debug("Creating collection: host={}, collection={}, config={}, numShards={}, numReplicas={}",
host, collectionName, configSet, 1, 1);
StringUtils.join(",", hosts), collectionName, configSet, 1, 1);
try {
CollectionAdminRequest request = CollectionAdminRequest.createCollection(collectionName, configSet, 1, 1);
request.process(solrClient);
Expand Down Expand Up @@ -353,11 +322,39 @@ public void close() throws IOException {
}
}

private boolean isCloud() {
if (StringUtils.isEmpty(mode)) {
logger.warn("Solr 'mode' is empty, setting default 'cloud'");
mode = "cloud";
public SolrClient newSolrClient(int timeout) {
final SolrClient solrClient;
if (hosts.get(0).startsWith("http")) {
if (hosts.size() == 1) {
// Single HTTP endpoint.
solrClient = new HttpSolrClient.Builder(hosts.get(0)).build();
((HttpSolrClient) solrClient).setRequestWriter(new BinaryRequestWriter());
((HttpSolrClient) solrClient).setSoTimeout(timeout);
} else {
// Use a LoadBalancer if there are multiple http hosts
solrClient = new LBHttpSolrClient.Builder().withBaseSolrUrls(hosts.toArray(new String[0])).build();

((LBHttpSolrClient) solrClient).setRequestWriter(new BinaryRequestWriter());
((LBHttpSolrClient) solrClient).setSoTimeout(timeout);
}
} else {
// If the provided hosts are not http, assume zookeeper hosts like HOST:PORT
// This client will use Zookeeper to discover Solr endpoints for SolrCloud collections, and then use the
// LBHttpSolrClient to issue requests.
if (isCloud()) {
solrClient = new CloudSolrClient.Builder().withZkHost(hosts).build();

((CloudSolrClient) solrClient).setRequestWriter(new BinaryRequestWriter());
((CloudSolrClient) solrClient).setSoTimeout(timeout);
} else {
throw new IllegalArgumentException("Can not initialize SolrManager from Zookeeper host not in Cloud mode");
}
}

return solrClient;
}

private boolean isCloud() {
switch (mode.toLowerCase()) {
case "collection":
case "cloud": {
Expand All @@ -376,37 +373,22 @@ private boolean isCloud() {
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("SolrManager{");
sb.append("host='").append(host).append('\'');
sb.append("hosts='").append(hosts).append('\'');
sb.append(", mode='").append(mode).append('\'');
sb.append(", solrClient=").append(solrClient);
sb.append('}');
return sb.toString();
}

public String getHost() {
return host;
}

public SolrManager setHost(String host) {
this.host = host;
return this;
public List<String> getHosts() {
return hosts;
}

public String getMode() {
return mode;
}

public SolrManager setMode(String mode) {
this.mode = mode;
return this;
}

public SolrClient getSolrClient() {
return solrClient;
}

public SolrManager setSolrClient(SolrClient solrClient) {
this.solrClient = solrClient;
return this;
}
}
Loading