From 38121963152a232fa8463415a93d09727aad68e3 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 2 Jul 2019 09:18:08 +0200 Subject: [PATCH] feat(triplestore): Update Lucene index on startup (#1362) * feat: Update Lucene index on startup. * feat(triplestore): Update Lucene index when an rdfs:label changes. --- .../webapi/app/ApplicationStateActor.scala | 26 ++++++++---- .../ApplicationStateMessages.scala | 8 +++- .../TriplestoreMessages.scala | 8 ++++ .../responders/v2/ResourcesResponderV2.scala | 10 +++++ .../http/HttpTriplestoreConnector.scala | 40 +++++++++++++------ 5 files changed, 71 insertions(+), 21 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/app/ApplicationStateActor.scala b/webapi/src/main/scala/org/knora/webapi/app/ApplicationStateActor.scala index 50a3a896e7..035295c40d 100644 --- a/webapi/src/main/scala/org/knora/webapi/app/ApplicationStateActor.scala +++ b/webapi/src/main/scala/org/knora/webapi/app/ApplicationStateActor.scala @@ -4,7 +4,7 @@ import akka.actor.{Actor, ActorLogging, ActorSelection, Timers} import org.knora.webapi._ import org.knora.webapi.messages.app.appmessages.AppState.AppState import org.knora.webapi.messages.app.appmessages._ -import org.knora.webapi.messages.store.triplestoremessages.{CheckRepositoryRequest, CheckRepositoryResponse, RepositoryStatus} +import org.knora.webapi.messages.store.triplestoremessages.{CheckRepositoryRequest, CheckRepositoryResponse, RepositoryStatus, SearchIndexUpdateRequest, SparqlUpdateResponse} import org.knora.webapi.messages.v2.responder.SuccessResponseV2 import org.knora.webapi.messages.v2.responder.ontologymessages.LoadOntologiesRequestV2 import org.knora.webapi.responders.RESPONDER_MANAGER_ACTOR_PATH @@ -65,15 +65,17 @@ class ApplicationStateActor extends Actor with Timers with ActorLogging { case AppState.Stopped => // do nothing case AppState.StartingUp => self ! SetAppState(AppState.WaitingForRepository) case AppState.WaitingForRepository => self ! CheckRepository() // check DB - case AppState.RepositoryReady => self ! SetAppState(AppState.CreatingCaches) + case AppState.RepositoryReady => self ! SetAppState(AppState.CreatingCaches) case AppState.CreatingCaches => self ! CreateCaches() - case AppState.CachesReady => self ! SetAppState(AppState.LoadingOntologies) - case AppState.LoadingOntologies if skipOntologies => self ! SetAppState(AppState.OntologiesReady) // skipping loading of ontologies + case AppState.CachesReady => self ! SetAppState(AppState.UpdatingSearchIndex) + case AppState.UpdatingSearchIndex => self ! UpdateSearchIndex() + case AppState.SearchIndexReady => self ! SetAppState(AppState.LoadingOntologies) + case AppState.LoadingOntologies if skipOntologies => self ! SetAppState(AppState.OntologiesReady) // skipping loading of ontologies case AppState.LoadingOntologies if !skipOntologies => self ! LoadOntologies() // load ontologies case AppState.OntologiesReady => self ! SetAppState(AppState.Running) case AppState.Running => printWelcomeMsg() case AppState.MaintenanceMode => // do nothing - case value => throw UnsupportedValueException(s"The value: $value is not supported.") + case other => throw UnsupportedValueException(s"The value: $other is not supported.") } } case GetAppState() => { @@ -130,15 +132,25 @@ class ApplicationStateActor extends Actor with Timers with ActorLogging { self ! SetAppState(AppState.CachesReady) } + case UpdateSearchIndex() => { + storeManager ! SearchIndexUpdateRequest() + } + + case SparqlUpdateResponse() => { + self ! SetAppState(AppState.SearchIndexReady) + } + /* load ontologies request */ case LoadOntologies() => { - responderManager ! LoadOntologiesRequestV2(KnoraSystemInstances.Users.SystemUser) + responderManager ! LoadOntologiesRequestV2(KnoraSystemInstances.Users.SystemUser) } /* load ontologies response */ - case SuccessResponseV2(msg) => { + case SuccessResponseV2(_) => { self ! SetAppState(AppState.OntologiesReady) } + + case other => throw UnexpectedMessageException(s"ApplicationStateActor received an unexpected message $other of type ${other.getClass.getCanonicalName}") } override def postStop(): Unit = { diff --git a/webapi/src/main/scala/org/knora/webapi/messages/app/appmessages/ApplicationStateMessages.scala b/webapi/src/main/scala/org/knora/webapi/messages/app/appmessages/ApplicationStateMessages.scala index b2449096a3..4e07b87cfe 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/app/appmessages/ApplicationStateMessages.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/app/appmessages/ApplicationStateMessages.scala @@ -106,6 +106,11 @@ case class CheckRepository() extends ApplicationStateRequest */ case class CreateCaches() extends ApplicationStateRequest +/** + * Message for updating the triplestore's full-text search index. Used only inside the actor itself. + */ +case class UpdateSearchIndex() extends ApplicationStateRequest + /** * Message for initiating loading of ontologies. Used only inside the actor itself. */ @@ -116,6 +121,7 @@ case class LoadOntologies() extends ApplicationStateRequest */ object AppState extends Enumeration { type AppState = Value - val Stopped, StartingUp, WaitingForRepository, RepositoryReady, CreatingCaches, CachesReady, LoadingOntologies, OntologiesReady, MaintenanceMode, Running = Value + val Stopped, StartingUp, WaitingForRepository, RepositoryReady, CreatingCaches, CachesReady, + UpdatingSearchIndex, SearchIndexReady, LoadingOntologies, OntologiesReady, MaintenanceMode, Running = Value } diff --git a/webapi/src/main/scala/org/knora/webapi/messages/store/triplestoremessages/TriplestoreMessages.scala b/webapi/src/main/scala/org/knora/webapi/messages/store/triplestoremessages/TriplestoreMessages.scala index 17ee8666bd..c7486ac35b 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/store/triplestoremessages/TriplestoreMessages.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/store/triplestoremessages/TriplestoreMessages.scala @@ -227,6 +227,14 @@ case class CheckRepositoryRequest() extends TriplestoreRequest */ case class CheckRepositoryResponse(repositoryStatus: RepositoryStatus, msg: String) +/** + * Updates the triplestore's full-text search index. + * + * @param subjectIri if a subject has changed, update the index for that subject. Otherwise, updates + * the index to add any subjects not yet indexed. + */ +case class SearchIndexUpdateRequest(subjectIri: Option[String] = None) extends TriplestoreRequest + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Components of messages diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v2/ResourcesResponderV2.scala b/webapi/src/main/scala/org/knora/webapi/responders/v2/ResourcesResponderV2.scala index 1173fd51fd..ef1a0be40b 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v2/ResourcesResponderV2.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v2/ResourcesResponderV2.scala @@ -341,6 +341,16 @@ class ResourcesResponderV2(responderData: ResponderData) extends ResponderWithSt case None => () } + + // If the resource's label was changed, update the full-text search index. + _ <- updateResourceMetadataRequestV2.maybeLabel match { + case Some(_) => + for { + _ <- (storeManager ? SearchIndexUpdateRequest(Some(updateResourceMetadataRequestV2.resourceIri))).mapTo[SparqlUpdateResponse] + } yield () + + case None => FastFuture.successful(()) + } } yield SuccessResponseV2("Resource metadata updated") } diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala index 1f3986f956..7600282c9a 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala @@ -128,6 +128,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging { case InsertTriplestoreContent(rdfDataObjects) => try2Message(sender(), insertDataIntoTriplestore(rdfDataObjects), log) case HelloTriplestore(msg) if msg == triplestoreType => sender ! HelloTriplestore(triplestoreType) case CheckRepositoryRequest() => try2Message(sender(), checkRepository(), log) + case SearchIndexUpdateRequest(subjectIri) => try2Message(sender(), updateLuceneIndex(subjectIri), log) case other => sender ! Status.Failure(UnexpectedMessageException(s"Unexpected message $other of type ${other.getClass.getCanonicalName}")) } @@ -346,6 +347,29 @@ class HttpTriplestoreConnector extends Actor with ActorLogging { } yield response } + /** + * Updates the Lucene full-text search index. + */ + private def updateLuceneIndex(subjectIri: Option[IRI] = None): Try[SparqlUpdateResponse] = { + val indexUpdateSparqlString = subjectIri match { + case Some(definedSubjectIri) => + // A subject's content has changed. Update the index for that subject. + s"""PREFIX luc: + |INSERT DATA { luc:fullTextSearchIndex luc:addToIndex <$definedSubjectIri> . } + """.stripMargin + + case None => + // Add new subjects to the index. + """PREFIX luc: + |INSERT DATA { luc:fullTextSearchIndex luc:updateIndex _:b1 . } + """.stripMargin + } + + for { + _ <- getTriplestoreHttpResponse(indexUpdateSparqlString, isUpdate = true) + } yield SparqlUpdateResponse() + } + /** * Performs a SPARQL update operation. * @@ -361,12 +385,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging { // If we're using GraphDB, update the full-text search index. _ = if (triplestoreType == TriplestoreTypes.HttpGraphDBSE | triplestoreType == TriplestoreTypes.HttpGraphDBFree) { - val indexUpdateSparqlString = - """ - PREFIX luc: - INSERT DATA { luc:fullTextSearchIndex luc:updateIndex _:b1 . } - """ - getTriplestoreHttpResponse(indexUpdateSparqlString, isUpdate = true) + updateLuceneIndex() } } yield SparqlUpdateResponse() } @@ -458,12 +477,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging { if (triplestoreType == TriplestoreTypes.HttpGraphDBSE || triplestoreType == TriplestoreTypes.HttpGraphDBFree) { /* need to update the lucene index */ - val indexUpdateSparqlString = - """ - PREFIX luc: - INSERT DATA { luc:fullTextSearchIndex luc:updateIndex _:b1 . } - """ - getTriplestoreHttpResponse(indexUpdateSparqlString, isUpdate = true) + updateLuceneIndex() } log.debug("==>> Loading Data End") @@ -610,7 +624,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging { } val took = System.currentTimeMillis() - start - log.info(s"[${statusCode}] GraphDB Query took: ${took}ms") + log.info(s"[$statusCode] GraphDB Query took: ${took}ms") responseEntityStr } finally {