Skip to content

Commit

Permalink
#101 Add a field to store document type
Browse files Browse the repository at this point in the history
  • Loading branch information
To-om committed Feb 19, 2021
1 parent 5c91fb3 commit fd2b798
Show file tree
Hide file tree
Showing 11 changed files with 33 additions and 37 deletions.
4 changes: 2 additions & 2 deletions app/org/elastic4play/database/DBCreate.scala
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,8 @@ class DBCreate @Inject() (db: DBConfiguration) {
}

private def addParent(modelName: String, parent: Option[BaseEntity], entity: JsObject): JsObject = parent match {
case Some(p) => entity + ("relations" -> Json.obj("name" -> modelName, "parent" -> p.id))
case None => entity + ("relations" -> JsString(modelName))
case Some(p) => entity + ("relations" -> Json.obj("name" -> modelName, "parent" -> p.id)) + ("docType" -> JsString(modelName))
case None => entity + ("docType" -> JsString(modelName))
}

/**
Expand Down
8 changes: 4 additions & 4 deletions app/org/elastic4play/database/DBIndex.scala
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,18 @@ class DBIndex(db: DBConfiguration, nbShards: Int, nbReplicas: Int, settings: Map
val mappingTemplates = Collection.distinctBy(models.flatMap(_.attributes).flatMap(_.elasticTemplate()))(_.name)
val fields = models.flatMap(_.attributes.filterNot(_.attributeName == "_id").map(_.elasticMapping)).toSeq
val relationsField = models
.map {
.collect {
case child: ChildModelDef[_, _, _, _] => child.parentModel.modelName -> Seq(child.modelName)
case model => model.modelName -> Nil
}
.groupBy(_._1)
.foldLeft(joinField("relations")) {
case (join, (parent, child)) => join.relation(parent, child.flatMap(_._2).toSeq)
}
val docTypeField = keywordField("docType")

for {
majorVersion <- nodeMajorVersion
modelMapping = properties(fields :+ relationsField)
modelMapping = properties(fields :+ relationsField :+ docTypeField)
.dateDetection(false)
.numericDetection(false)
.templates(mappingTemplates)
Expand Down Expand Up @@ -101,7 +101,7 @@ class DBIndex(db: DBConfiguration, nbShards: Int, nbReplicas: Int, settings: Map
*/
def getSize(modelName: String)(implicit ec: ExecutionContext): Future[Long] =
db.execute {
search(db.indexName).matchQuery("relations", modelName).size(0)
search(db.indexName).matchQuery("docType", modelName).size(0)
}
.map {
_.totalHits
Expand Down
2 changes: 1 addition & 1 deletion app/org/elastic4play/database/DBSequence.scala
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class DBSequence @Inject() (db: DBConfiguration) {
def apply(seqId: String)(implicit ec: ExecutionContext): Future[Int] =
db.execute {
updateById(db.indexName, s"sequence_$seqId")
.upsert("sequenceCounter" -> 1, "relations" -> "sequence")
.upsert("sequenceCounter" -> 1, "docType" -> "sequence")
.script("ctx._source.sequenceCounter += 1")
.retryOnConflict(5)
.fetchSource(true)
Expand Down
10 changes: 5 additions & 5 deletions app/org/elastic4play/database/DBUtils.scala
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ object DBUtils {
def hit2json(hit: SearchHit): JsObject = {
val id = JsString(hit.id)
val body = Json.parse(hit.sourceAsString).as[JsObject]
val (parent, model) = (body \ "relations" \ "parent").asOpt[JsString] match {
case Some(p) => p -> (body \ "relations" \ "name").as[JsString]
case None => JsNull -> (body \ "relations").as[JsString]
}
body - "relations" +
// Retrieve document type from the field "relations" for index pre-ES7.11
// This method is used by migration. "relations" must be used as fall-back to permit migration from pre-ES7.11
val model = (body \ "docType").asOpt[JsString].orElse((body \ "relations" \ "name").asOpt[JsString]).getOrElse((body \ "relations").as[JsString])
val parent = (body \ "relations" \ "parent").asOpt[JsString].getOrElse(JsNull)
body - "relations" - "docType" +
("_type" -> model) +
("_routing" -> hit.routing.fold(id)(JsString.apply)) +
("_parent" -> parent) +
Expand Down
12 changes: 5 additions & 7 deletions app/org/elastic4play/services/Aggregations.scala
Original file line number Diff line number Diff line change
Expand Up @@ -146,13 +146,11 @@ class SelectTop(aggregationName: String, size: Int, sortBy: Seq[String], query:

def processResult(model: BaseModelDef, aggregations: HasAggregations): JsObject = {
val hits = aggregations.result[TopHits](aggregationName).hits.map { hit =>
val id = JsString(hit.id)
val body = Json.parse(JacksonSupport.mapper.writeValueAsString(hit.source)).as[JsObject]
val (parent, model) = (body \ "relations" \ "parent").asOpt[JsString] match {
case Some(p) => p -> (body \ "relations" \ "name").as[JsString]
case None => JsNull -> (body \ "relations").as[JsString]
}
body - "relations" +
val id = JsString(hit.id)
val body = Json.parse(JacksonSupport.mapper.writeValueAsString(hit.source)).as[JsObject]
val model = (body \ "docType").as[JsString]
val parent = (body \ "relations" \ "parent").asOpt[JsString].getOrElse(JsNull)
body - "relations" - "docType" +
("_type" -> model) +
("_parent" -> parent) +
("_id" -> id)
Expand Down
2 changes: 1 addition & 1 deletion app/org/elastic4play/services/AttachmentSrv.scala
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ class AttachmentSrv(
}

def cleanup(implicit ec: ExecutionContext): Future[Unit] =
dbFind(Some("all"), Nil)(index => search(index).matchQuery("relations", attachmentModel.modelName).fetchSource(false))
dbFind(Some("all"), Nil)(index => search(index).matchQuery("docType", attachmentModel.modelName).fetchSource(false))
._1
.mapConcat(o => (o \ "_id").asOpt[String].toList)
.collect { case id if id.endsWith("_0") => id.dropRight(2) }
Expand Down
21 changes: 10 additions & 11 deletions app/org/elastic4play/services/FindSrv.scala
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class FindSrv @Inject() (dbfind: DBFind, modelSrv: ModelSrv) {
range: Option[String],
sortBy: Seq[String]
)(implicit ec: ExecutionContext): (Source[BaseEntity, NotUsed], Future[Long]) = {
val query = modelName.fold(queryDef)(m => and("relations" ~= m, queryDef)).query
val query = modelName.fold(queryDef)(m => and("docType" ~= m, queryDef)).query
val (src, total) = dbfind(range, sortBy)(indexName => search(indexName).query(query))
val entities = src.map { attrs =>
modelName match {
Expand All @@ -44,7 +44,7 @@ class FindSrv @Inject() (dbfind: DBFind, modelSrv: ModelSrv) {
def apply(model: BaseModelDef, queryDef: QueryDef, range: Option[String], sortBy: Seq[String])(
implicit ec: ExecutionContext
): (Source[BaseEntity, NotUsed], Future[Long]) = {
val (src, total) = dbfind(range, sortBy)(indexName => search(indexName).query(and("relations" ~= model.modelName, queryDef).query))
val (src, total) = dbfind(range, sortBy)(indexName => search(indexName).query(and("docType" ~= model.modelName, queryDef).query))
val entities = src.map(attrs => model(attrs))
(entities, total)
}
Expand All @@ -55,18 +55,17 @@ class FindSrv @Inject() (dbfind: DBFind, modelSrv: ModelSrv) {
range: Option[String],
sortBy: Seq[String]
)(implicit ec: ExecutionContext): (Source[E, NotUsed], Future[Long]) = {
val (src, total) = dbfind(range, sortBy)(indexName => search(indexName).query(and("relations" ~= model.modelName, queryDef).query))
val (src, total) = dbfind(range, sortBy)(indexName => search(indexName).query(and("docType" ~= model.modelName, queryDef).query))
val entities = src.map(attrs => model(attrs))
(entities, total)
}

def apply(model: BaseModelDef, queryDef: QueryDef, aggs: Agg*)(implicit ec: ExecutionContext): Future[JsObject] =
dbfind(indexName =>
search(indexName).query(and("relations" ~= model.modelName, queryDef).query).aggregations(aggs.flatMap(_.apply(model))).size(0)
).map { searchResponse =>
aggs
.map(_.processResult(model, searchResponse.aggregations))
.reduceOption(_ ++ _)
.getOrElse(JsObject.empty)
}
dbfind(indexName => search(indexName).query(and("docType" ~= model.modelName, queryDef).query).aggregations(aggs.flatMap(_.apply(model))).size(0))
.map { searchResponse =>
aggs
.map(_.processResult(model, searchResponse.aggregations))
.reduceOption(_ ++ _)
.getOrElse(JsObject.empty)
}
}
3 changes: 1 addition & 2 deletions app/org/elastic4play/services/MigrationSrv.scala
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,7 @@ class MigrationSrv @Inject() (
.zipWith(count) { (entity, current) =>
eventSrv.publish(MigrationEvent(modelName, current.toLong, total))
(entity \ "_type").asOpt[JsString].fold(entity) { t =>
val relations = (entity \ "_parent").asOpt[JsString].fold[JsValue](t)(p => Json.obj("name" -> t, "parent" -> p))
entity - "_type" - "_parent" + ("relations" -> relations)
(entity \ "_parent").asOpt[JsString].fold(entity - "_type" + ("docType" -> t))(p => entity - "_type" - "_parent" + ("relations" -> Json.obj("name" -> t, "parent" -> p)))
}
}
.runWith(dbcreate.sink())
Expand Down
4 changes: 2 additions & 2 deletions app/org/elastic4play/services/QueryDSL.scala
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ object QueryDSL {
new GroupByCategory(aggregationName.getOrElse("categories"), categories, selectables)

private def nestedField(field: String, q: String => Query) = field match {
case "_type" => q("relations")
case "_type" => q("docType")
case _ =>
field
.split("\\.")
Expand Down Expand Up @@ -90,7 +90,7 @@ object QueryDSL {
def in(values: String*): QueryDef = QueryDef(nestedField(field, termsQuery(_, values)))
}

def ofType(value: String): QueryDef = QueryDef(termQuery("relations", value))
def ofType(value: String): QueryDef = QueryDef(termQuery("docType", value))
def withId(entityIds: String*): QueryDef = QueryDef(idsQuery(entityIds))
def any: QueryDef = QueryDef(matchAllQuery)
def contains(field: String): QueryDef = QueryDef(nestedField(field, existsQuery))
Expand Down
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ scalaVersion := "2.12.12"

resolvers += "elasticsearch-releases" at "https://artifacts.elastic.co/maven"

val elastic4sVersion = "7.9.1"
val elastic4sVersion = "7.10.0"
libraryDependencies ++= Seq(
cacheApi,
"com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sVersion,
Expand Down
2 changes: 1 addition & 1 deletion version.sbt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version := "1.12.3"
version := "1.13.0-SNAPSHOT"

0 comments on commit fd2b798

Please sign in to comment.