Skip to content
This repository has been archived by the owner on Apr 28, 2023. It is now read-only.

Commit

Permalink
issue-550: fix duplicate Individuals issue (#574)
Browse files Browse the repository at this point in the history 
* issue-550: fix indexing duplicate Individuals

fix for #550

* issue-550: display anonymous type for individuals

related fix for #550

* issue-550: fix indexing duplication for related Individuals

fix for #550

* issue-550: rename anonymous type key

related fix for #550

* issue-550: fix indexing duplicate Individuals

fix for #550

* issue-550: display anonymous type for individuals

related fix for #550

* issue-550: fix indexing duplication for related Individuals

fix for #550

* issue-550: rename anonymous type key

related fix for #550

Co-authored-by: Josh Lagrimas <joshlagrimas@ebi.ac.uk>
  • Loading branch information
@serjoshua
serjoshua and Josh Lagrimas authored Apr 7, 2022
1 parent 7d7456f commit 568cab7
Show file tree
Hide file tree
Showing 9 changed files with 952 additions and 909 deletions.
77 changes: 36 additions & 41 deletions ols-neo4j/src/main/java/uk/ac/ebi/spot/ols/loader/BatchNeo4JIndexer.java
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,6 @@
package uk.ac.ebi.spot.ols.loader;

import static uk.ac.ebi.spot.ols.loader.Neo4JIndexerConstants.*;
import static uk.ac.ebi.spot.ols.config.OntologyDefaults.THING;

import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import org.neo4j.graphdb.DynamicLabel;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Label;
import org.neo4j.graphdb.Result;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.*;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.graphdb.index.IndexHits;
import org.neo4j.graphdb.schema.IndexDefinition;
Expand All @@ -30,12 +15,17 @@
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import uk.ac.ebi.spot.ols.config.OlsNeo4jConfiguration;
import uk.ac.ebi.spot.ols.exception.IndexingException;
import uk.ac.ebi.spot.ols.model.OntologyIndexer;
import uk.ac.ebi.spot.ols.util.LocalizedStrings;

import java.util.*;
import java.util.concurrent.TimeUnit;

import static uk.ac.ebi.spot.ols.config.OntologyDefaults.THING;
import static uk.ac.ebi.spot.ols.loader.Neo4JIndexerConstants.*;

/**
* @author Simon Jupp
* @date 17/06/2015
Expand Down Expand Up @@ -110,7 +100,7 @@ private Long getOrCreateMergedNode(BatchInserter inserter, Map<String, Long> mer
}
}

properties.put("label", labels.getFirstString("en"));
properties.put("label", labels.getFirstString("en"));

Long hit = inserter.createNode(properties, nodeLabel);
index.add(hit, properties);
Expand Down Expand Up @@ -220,6 +210,11 @@ private void indexIndividuals(BatchInserter inserter, OntologyLoader loader,

for (IRI individualIri : loader.getAllIndividualIRIs()) {

// avoid duplicating individuals already related to a class
if (classNodeMap.containsKey(individualIri.toString())) {
nodeMap.put(individualIri.toString(), classNodeMap.get(individualIri.toString()));
}

Long node = NodeCreator.getOrCreateNode(inserter, nodeMap, loader, individualIri,
new LinkedList<Label>(Arrays.asList(instanceLabel, _instanceLabel,
nodeOntologyLabel)));
Expand Down Expand Up @@ -250,14 +245,19 @@ private void indexIndividuals(BatchInserter inserter, OntologyLoader loader,
}

// add relations
indexRelations(node, loader.getRelatedIndividuals(individualIri),
inserter, loader, nodeMap,
new LinkedList<Label>(Arrays.asList(instanceLabel, nodeOntologyLabel, _instanceLabel)));

indexRelations(node, loader.getRelatedClassesToIndividual(individualIri),
inserter, loader, classNodeMap,
new LinkedList<Label>(Arrays.asList(nodeLabel, nodeOntologyLabel, _nodeLabel)));
}

// add related individuals only after indexing all individuals to avoid duplication
for (IRI individualIri : loader.getAllIndividualIRIs()) {
if (loader.getRelatedIndividuals(individualIri).size() > 0) {
indexRelations(nodeMap.get(individualIri.toString()), loader.getRelatedIndividuals(individualIri),
inserter, loader, nodeMap,
new LinkedList<Label>(Arrays.asList(instanceLabel, nodeOntologyLabel, _instanceLabel)));
}
}
}

private void indexRelations(Long node, Map<IRI, Collection<IRI>> relatedIndividuals,
Expand Down Expand Up @@ -429,7 +429,7 @@ private static void addLocalizedProperties(

properties.put(propertyName, localizedStrings.getFirstString("en"));

for(String language : localizedStrings.getLanguages()) {
for (String language : localizedStrings.getLanguages()) {
properties.put(propertyName + "_" + language, localizedStrings.getFirstString(language));
}
}
Expand All @@ -449,7 +449,7 @@ public void createIndex(Collection<OntologyLoader> loaders) throws IndexingExcep
for (OntologyLoader loader : loaders) {

BatchInserter inserter = getBatchIndexer(loader.getOntologyName());

setOntologyLabel(loader.getOntologyName());
// index classes
indexClasses(inserter, loader, classNodeMap, mergedNodeMap);
Expand Down Expand Up @@ -492,27 +492,24 @@ public void createIndex(Collection<OntologyLoader> loaders) throws IndexingExcep
} catch (IllegalStateException e) {
throw new IndexingException("Building Neo4j index failed as the schema index didn't finish in time", e);
}
}
else if (state.equals(Schema.IndexState.FAILED)) {
} else if (state.equals(Schema.IndexState.FAILED)) {
throw new Exception("Index failed: " + indexDefinition.getLabel().name());
}
}

tx.success();
}
catch (Exception e) {
logger.debug(e.getMessage(), e);
} catch (Exception e) {
logger.debug(e.getMessage(), e);
tx.failure();
throw new IndexingException("Building Neo4j index failed as the schema index creation failed", e);
}
finally {
} finally {
tx.close();
db.shutdown();
}
}

protected GraphDatabaseService getGraphDatabase () {
return new GraphDatabaseFactory().newEmbeddedDatabase(neo4jConfiguration.getNeo4JPath());
protected GraphDatabaseService getGraphDatabase() {
return new GraphDatabaseFactory().newEmbeddedDatabase(neo4jConfiguration.getNeo4JPath());
}

public void dropIndex(OntologyLoader loader) throws IndexingException {
Expand All @@ -538,14 +535,14 @@ private void deleteNodes(String ontologyName) {
int count = getNodeCount(
"match (n:" + ontologyName.toUpperCase() + ")-[r]->() return count(r) as count", ontologyName);

for (int x = 0; x < count ; x +=DELETE_SIZE) {
for (int x = 0; x < count; x += DELETE_SIZE) {

Transaction tx = db.beginTx();

try {
String cypherDelete =
"match (n:" + ontologyName.toUpperCase() + ")-[r]->() with r limit " +
DELETE_SIZE + " delete r";
"match (n:" + ontologyName.toUpperCase() + ")-[r]->() with r limit " +
DELETE_SIZE + " delete r";
getLogger().info("executing delete: " + cypherDelete);
Result result = db.execute(cypherDelete);
getLogger().info(result.resultAsString());
Expand All @@ -561,7 +558,7 @@ private void deleteNodes(String ontologyName) {
count = getNodeCount(
"match (n:" + ontologyName.toUpperCase() + ") return count(n) as count", ontologyName
);
for (int x = 0; x < count ; x +=DELETE_SIZE) {
for (int x = 0; x < count; x += DELETE_SIZE) {

Transaction tx = db.beginTx();

Expand Down Expand Up @@ -593,12 +590,10 @@ private int getNodeCount(String nodeCountCypher, String ontologyName) {
count = (Long) result.next().get("count");
getLogger().debug("query count " + count);
tx.success();
}
catch (Exception e) {
} catch (Exception e) {
tx.failure();
throw new IndexingException("Couldn't count: " + ontologyName, e);
}
finally {
} finally {
tx.close();
}
return count.intValue();
Expand Down
Loading

0 comments on commit 568cab7

Please sign in to comment.