Skip to content

Commit

Permalink
Set up integration testing environment
Browse files Browse the repository at this point in the history
* requires the Docker Maven Plugin be compiled and installed (mvn install) from https://github.com/emetsger/docker-maven-plugin/tree/volumes-846 (see also: fabric8io/docker-maven-plugin#846)
* configures an "http-solr" bean profile, with a corresponding http-solr.properties file
* starts up solr in pre-integration-test, stops it in post-integration-test
* configures the failsafe plugin to run integration tests
* adds a SimpleSolrIT
* ignores the SimpleSolrTest (any methods that are still needed should be moved to SimpleSolrIT, because they require a running Solr instance as they are currently written)
  • Loading branch information
emetsger committed Oct 19, 2017
1 parent 76da968 commit 766b003
Show file tree
Hide file tree
Showing 7 changed files with 236 additions and 126 deletions.
37 changes: 36 additions & 1 deletion indexing-solr/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

<build>
<plugins>

<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
Expand All @@ -30,7 +31,6 @@
<time>120000</time>
</wait>
</run>

<external>
<type>compose</type>
<basedir>src/test/docker</basedir>
Expand All @@ -39,7 +39,42 @@
</image>
</images>
</configuration>
<executions>
<execution>
<id>start-docker-its</id>
<phase>pre-integration-test</phase>
<goals>
<goal>start</goal>
</goals>
</execution>
<execution>
<id>stop-docker-its</id>
<phase>post-integration-test</phase>
<goals>
<goal>stop</goal>
</goals>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<systemProperties>
<rmap.solr.url>http://${docker.host.address}:8983/solr</rmap.solr.url>
</systemProperties>
</configuration>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>

</plugins>
</build>

Expand Down
1 change: 1 addition & 0 deletions indexing-solr/src/main/resources/http-solr.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
rmap.solr.url = http://localhost:8983/solr
27 changes: 16 additions & 11 deletions indexing-solr/src/main/resources/rmap-indexing-solr.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,28 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:solr="http://www.springframework.org/schema/data/solr"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/data/solr http://www.springframework.org/schema/data/solr/spring-solr-2.0.xsd">
http://www.springframework.org/schema/data/solr http://www.springframework.org/schema/data/solr/spring-solr-2.0.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

<solr:repositories base-package="info.rmapproject.indexing.solr" multicore-support="true"/>

<solr:solr-client id="solrClient" url="http://192.168.99.100:8983/solr"/>
<!--<context:property-placeholder location="classpath:/solr.properties" ignore-unresolvable="true"/>-->

<bean id="standAloneStatusInferencer" class="info.rmapproject.indexing.solr.repository.StandAloneStatusInferencer"/>

<bean id="indexableThingMapper" class="info.rmapproject.indexing.solr.repository.SimpleIndexableThingMapper"/>
<solr:repositories base-package="info.rmapproject.indexing.solr" multicore-support="true"/>

<bean id="agentMapper" class="info.rmapproject.indexing.solr.repository.SimpleAgentMapper"/>

<bean id="eventMapper" class="info.rmapproject.indexing.solr.repository.SimpleEventMapper"/>
<beans profile="http-solr">
<context:property-placeholder location="classpath*:/http-solr.properties" ignore-unresolvable="true"/>
<solr:solr-client id="solrClient" url="${rmap.solr.url}"/>
</beans>

<bean id="discoMapper" class="info.rmapproject.indexing.solr.repository.SimpleDiscoMapper"/>
<beans>
<bean id="standAloneStatusInferencer" class="info.rmapproject.indexing.solr.repository.StandAloneStatusInferencer"/>

<bean id="indexDTOMapper" class="info.rmapproject.indexing.solr.repository.SimpleIndexDTOMapper"/>
<bean id="indexableThingMapper" class="info.rmapproject.indexing.solr.repository.SimpleIndexableThingMapper"/>
<bean id="agentMapper" class="info.rmapproject.indexing.solr.repository.SimpleAgentMapper"/>
<bean id="eventMapper" class="info.rmapproject.indexing.solr.repository.SimpleEventMapper"/>
<bean id="discoMapper" class="info.rmapproject.indexing.solr.repository.SimpleDiscoMapper"/>
<bean id="indexDTOMapper" class="info.rmapproject.indexing.solr.repository.SimpleIndexDTOMapper"/>
</beans>

</beans>
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* @author Elliot Metsger (emetsger@jhu.edu)
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ActiveProfiles({"default", "inmemory-triplestore", "inmemory-idservice"})
@ActiveProfiles({"default", "inmemory-triplestore", "inmemory-idservice", "http-solr"})
@ContextConfiguration({"classpath:/rmap-indexing-solr.xml", "classpath:/spring-rmapcore-context.xml"})
public abstract class AbstractSpringIndexingTest {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,14 @@
import info.rmapproject.core.model.RMapIri;
import info.rmapproject.core.model.RMapObject;
import info.rmapproject.core.model.RMapObjectType;
import info.rmapproject.core.model.agent.RMapAgent;
import info.rmapproject.core.model.disco.RMapDiSCO;
import info.rmapproject.core.model.event.RMapEvent;
import info.rmapproject.core.rdfhandler.RDFHandler;
import info.rmapproject.core.rdfhandler.RDFType;
import info.rmapproject.core.rdfhandler.impl.openrdf.RioRDFHandler;
import info.rmapproject.indexing.solr.repository.IndexDTO;
import info.rmapproject.indexing.solr.repository.SimpleSolrTest;
import org.openrdf.model.IRI;
import org.openrdf.model.Statement;
import org.openrdf.rio.RDFFormat;
Expand All @@ -21,13 +26,19 @@
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.channels.ReadableByteChannel;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static info.rmapproject.indexing.solr.IndexUtils.findEventIri;
import static info.rmapproject.indexing.solr.IndexUtils.irisEqual;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
Expand Down Expand Up @@ -184,7 +195,78 @@ public static <T extends RMapObject> List<T> getRmapObjects(Map<RMapObjectType,
}

/**
* A Spring Resource of RDF content. A RDFResource exposes the RDF serialization of the RDF.
* Creates an {@link IndexDTO} for each {@link RMapEvent} found at the specified {@code resourcePath}.
* <h3>Assumptions:</h3>
* <ul>
* <li>{@code resourcePath} names a classpath resource that resolves to a directory on the filesystem. The
* directory contains RMap objects serialized in N-Quads format. Each file contains one RMap object, and
* ends with an {@code .n4} file extension</li>
* <li>The supplied {@code rdfHandler} is able to de-serialize N-Quad RDF</li>
* <li>Each {@code RMapEvent} under {@code resourcePath} forms a <em>connected graph</em>, a requirement of the
* {@link IndexDTO}. That means that with every event, there ought to be an object for the agent, and at
* least one source and/or target disco.</li>
* </ul>
* <p>
* The {@code assertions} are applied on the deserialized objects prior to preparing the {@code IndexDTO} stream.
* The caller may use these assertions to verify the type, number, or contents of the RMap objects that have been
* deserialized from the filesystem.
* </p>
*
* @param rdfHandler the RDFHandler capable of deserializing N-Quads
* @param resourcePath a classpath resource that is expected to resolve to a directory containing RDF serializations
* of RMap objects in N-Quad format. One RMap object per file.
* @param assertions caller-supplied assertions that are run on the de-serialized RDF prior to assembling
* {@code IndexDTO} objects
* @return a stream of {@code IndexDTO} objects
*/
public static Stream<IndexDTO> prepareIndexableDtos(RDFHandler rdfHandler, String resourcePath,
Consumer<Map<RMapObjectType, Set<RDFResource>>> assertions) {
Map<RMapObjectType, Set<RDFResource>> rmapObjects = new HashMap<>();
getRmapResources(resourcePath, rdfHandler, RDFFormat.NQUADS, rmapObjects);

assertions.accept(rmapObjects);

List<RMapDiSCO> discos = getRmapObjects(rmapObjects, RMapObjectType.DISCO, rdfHandler);
List<RMapEvent> events = getRmapObjects(rmapObjects, RMapObjectType.EVENT, rdfHandler);
List<RMapAgent> agents = getRmapObjects(rmapObjects, RMapObjectType.AGENT, rdfHandler);

return events.stream()
.sorted(Comparator.comparing(RMapEvent::getStartTime))
.map(event -> {
RMapAgent agent = agents.stream()
.filter(a -> irisEqual(a.getId(), event.getAssociatedAgent()))
.findAny()
.orElseThrow(() ->
new RuntimeException("Missing expected agent " +
event.getAssociatedAgent().getStringValue()));

Optional<RMapIri> source = findEventIri(event, IndexUtils.EventDirection.SOURCE);
Optional<RMapIri> target = findEventIri(event, IndexUtils.EventDirection.TARGET);
RMapDiSCO sourceDisco = null;
RMapDiSCO targetDisco = null;

if (source.isPresent()) {
sourceDisco = discos.stream()
.filter(disco -> disco.getId().getStringValue().equals(source.get().getStringValue()))
.findAny().get();
}

if (target.isPresent()) {
targetDisco = discos.stream()
.filter(disco -> disco.getId().getStringValue().equals(target.get().getStringValue()))
.findAny().get();
}

IndexDTO indexDto = new IndexDTO(event, agent, sourceDisco, targetDisco);

return indexDto;
});
}

/**
* A Spring Resource of RDF content. A {@code RDFResource} exposes the RDF serialization of the RDF, along with an
* {@code InputStream} to the content. This allows callers to determine which serialization to use when
* deserializing the content of the resource.
*/
public interface RDFResource extends Resource {

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package info.rmapproject.indexing.solr.repository;

import info.rmapproject.core.model.RMapObjectType;
import info.rmapproject.core.model.agent.RMapAgent;
import info.rmapproject.core.model.disco.RMapDiSCO;
import info.rmapproject.core.model.event.RMapEvent;
import info.rmapproject.core.rdfhandler.RDFHandler;
import info.rmapproject.indexing.solr.AbstractSpringIndexingTest;
import info.rmapproject.indexing.solr.TestUtils;
import info.rmapproject.indexing.solr.model.DiscoSolrDocument;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Stream;

import static info.rmapproject.core.model.RMapStatus.ACTIVE;
import static info.rmapproject.core.model.RMapStatus.INACTIVE;
import static info.rmapproject.indexing.solr.TestUtils.prepareIndexableDtos;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

/**
* @author Elliot Metsger (emetsger@jhu.edu)
*/
public class SimpleSolrIT extends AbstractSpringIndexingTest {

@Autowired
private RDFHandler rdfHandler;

@Autowired
private DiscoRepository discoRepository;

/**
* Tests the {@link DiscoRepository#index(IndexDTO)} method by supplying three {@code IndexDTO}s for indexing from
* the {@code /data/discos/rmd18mddcw} directory:
* <ul>
* <li>a creation event</li>
* <li>followed by an update event</li>
* <li>followed by another update event</li>
* </ul>
* This test insures that the {@link DiscoSolrDocument}s in the index have the correct
* {@link DiscoSolrDocument#DISCO_STATUS} after the three events have been processed.
*
* @see <a href="src/test/resources/data/discos/rmd18mddcw/README.md">README.MD</a>
*/
@Test
public void testIndexingDiscoStatusCreateAndUpdateAndUpdate() {
LOG.debug("Deleting everything in the index.");
discoRepository.deleteAll();
assertEquals(0, discoRepository.count());

Consumer<Map<RMapObjectType, Set<TestUtils.RDFResource>>> assertions = (resources) -> {
List<RMapDiSCO> discos = TestUtils.getRmapObjects(resources, RMapObjectType.DISCO, rdfHandler);
assertEquals(3, discos.size());

List<RMapEvent> events = TestUtils.getRmapObjects(resources, RMapObjectType.EVENT, rdfHandler);
assertEquals(3, events.size());

List<RMapAgent> agents = TestUtils.getRmapObjects(resources, RMapObjectType.AGENT, rdfHandler);
assertEquals(1, agents.size());
};

LOG.debug("Preparing indexable objects.");
Stream<IndexDTO> dtos = prepareIndexableDtos(rdfHandler,"/data/discos/rmd18mddcw", assertions);

dtos.peek(dto -> LOG.debug("Indexing {}", dto)).forEach(dto -> discoRepository.index(dto));

// 5 documents should have been added
// - one document per DiSCO, Event tuple
assertEquals(5, discoRepository.count());

// 1 document should be active
Set<DiscoSolrDocument> docs = discoRepository.findDiscoSolrDocumentsByDiscoStatus(ACTIVE.toString());
assertEquals(1, docs.size());

// assert it is the uri we expect
DiscoSolrDocument active = docs.iterator().next();
assertTrue(active.getDiscoUri().endsWith("rmd18mddcw"));

// the other four should be inactive
docs = discoRepository.findDiscoSolrDocumentsByDiscoStatus(INACTIVE.toString());
assertEquals(4, docs.size());

// assert they have the uris we expect
assertEquals(2, docs.stream().filter(doc -> doc.getDiscoUri().endsWith("rmd18mdd8b")).count());
assertEquals(2, docs.stream().filter(doc -> doc.getDiscoUri().endsWith("rmd18m7mr7")).count());
}


}
Loading

0 comments on commit 766b003

Please sign in to comment.