#368 migrate route relations documents to new format

server/src/main/scala/kpn/api/common/data/Way.scala

@@ -10,7 +10,8 @@ case class Way(
  timestamp: Timestamp,
  changeSetId: Long,
  tags: Tags,
- nodes: Vector[Node], length: Long /* meters */
+ nodes: Vector[Node],
+ length: Long /* meters */
 ) extends Element {
  override def isWay: Boolean = true
 

server/src/main/scala/kpn/api/custom/ScopedNetworkType.scala

@@ -24,14 +24,9 @@ object ScopedNetworkType {
  all.find(_.key == key)
  }
 
- def withNetworkType(networkType: NetworkType): Seq[ScopedNetworkType] = {
- all.filter(_.networkType == networkType)
- }
-
  def from(networkScope: NetworkScope, networkType: NetworkType): ScopedNetworkType = {
  all.find(ns => ns.networkType == networkType && ns.networkScope == networkScope).get
  }
-
 }
 
 case class ScopedNetworkType(networkScope: NetworkScope, networkType: NetworkType, key: String) {
@@ -47,5 +42,4 @@ case class ScopedNetworkType(networkScope: NetworkScope, networkType: NetworkTyp
  def proposedNodeNameTagKey: String = s"proposed:${key}_name"
 
  def expectedRouteRelationsTag: String = "expected_" + key + "_route_relations"
-
 }

server/src/main/scala/kpn/core/tools/next/domain/OldMember.scala

@@ -0,0 +1,12 @@
+package kpn.core.tools.next.domain
+
+trait OldMember {
+
+ def role: Option[String]
+
+ def isNode: Boolean = false
+
+ def isWay: Boolean = false
+
+ def isRelation: Boolean = false
+}

server/src/main/scala/kpn/core/tools/next/domain/OldNode.scala

@@ -0,0 +1,5 @@
+package kpn.core.tools.next.domain
+
+import kpn.api.common.data.raw.RawNode
+
+case class OldNode(raw: RawNode)

server/src/main/scala/kpn/core/tools/next/domain/OldNodeMember.scala

@@ -0,0 +1,5 @@
+package kpn.core.tools.next.domain
+
+case class OldNodeMember(node: OldNode, role: Option[String]) extends OldMember {
+ override def isNode: Boolean = true
+}

server/src/main/scala/kpn/core/tools/next/domain/OldRelation.scala

@@ -1,9 +1,8 @@
 package kpn.core.tools.next.domain
 
-import kpn.api.common.data.Member
 import kpn.api.common.data.raw.RawRelation
 
 case class OldRelation(
  raw: RawRelation,
- members: Seq[Member]
+ members: Seq[OldMember]
 )

server/src/main/scala/kpn/core/tools/next/domain/OldRelationMember.scala

@@ -0,0 +1,5 @@
+package kpn.core.tools.next.domain
+
+case class OldRelationMember(relation: OldRelation, role: Option[String]) extends OldMember {
+ override def isRelation: Boolean = true
+}

server/src/main/scala/kpn/core/tools/next/domain/OldWay.scala

@@ -0,0 +1,5 @@
+package kpn.core.tools.next.domain
+
+import kpn.api.common.data.raw.RawWay
+
+case class OldWay(raw: RawWay, nodes: Seq[OldNode], length: Long)

server/src/main/scala/kpn/core/tools/next/domain/OldWayMember.scala

@@ -0,0 +1,5 @@
+package kpn.core.tools.next.domain
+
+case class OldWayMember(way: OldWay, role: Option[String]) extends OldMember {
+ override def isWay: Boolean = true
+}

server/src/main/scala/kpn/core/tools/next/support/NextExploreRelationsTool.scala → server/src/main/scala/kpn/core/tools/next/support/CollectRouteRelationIdsTool.scala

@@ -8,11 +8,11 @@ import org.apache.commons.io.FileUtils
 import java.io.File
 import scala.xml.XML
 
-object MonitorExploreRelationsTool {
+object CollectRouteRelationIdsTool {
  def main(args: Array[String]): Unit = {
  val overpassQueryExecutor = new OverpassQueryExecutorRemoteImpl()
- val tool = new MonitorExploreRelationsTool(overpassQueryExecutor)
- tool.explore()
+ val tool = new CollectRouteRelationIdsTool(overpassQueryExecutor)
+ tool.collect()
  }
 }
 
@@ -25,8 +25,8 @@ case class RouteInfo(
  def routeIds: Seq[Long] = nodeNetworkRouteIds ++ noneNodeNetworkRouteIds
 }
 
-class MonitorExploreRelationsTool(overpassQueryExecutor: OverpassQueryExecutor) {
- def explore(): Unit = {
+class CollectRouteRelationIdsTool(overpassQueryExecutor: OverpassQueryExecutor) {
+ def collect(): Unit = {
  val routeInfos = queryRouteInfos()
  report(routeInfos)
  saveRouteIds(routeInfos)
@@ -80,11 +80,11 @@ class MonitorExploreRelationsTool(overpassQueryExecutor: OverpassQueryExecutor)
  val queryStart = s"[date:'2024-03-01T00:00:00Z'];relation['type'='$typeTagValue']['route'='$routeTagValue']"
  val nodeNetworkRouteIds = {
  val query = s"$queryStart['network:type'='node_network'];out ids;"
- queryRouteIds(typeTagValue, routeTagValue, query)
+ queryRouteIds(query)
  }
  val noneNodeNetworkRouteIds = {
  val query = s"$queryStart['network:type'!='node_network'];out ids;"
- queryRouteIds(typeTagValue, routeTagValue, query)
+ queryRouteIds(query)
  }
  RouteInfo(
  typeTagValue,
@@ -94,7 +94,7 @@ class MonitorExploreRelationsTool(overpassQueryExecutor: OverpassQueryExecutor)
  )
  }
 
- private def queryRouteIds(typeTagValue: String, routeTagValue: String, query: String): Seq[Long] = {
+ private def queryRouteIds(query: String): Seq[Long] = {
  val xmlString = overpassQueryExecutor.execute(query)
  val xml = XML.loadString(xmlString)
  (xml.head \ "relation").map(r => (r \ "@id").text).map(_.toLong)

server/src/main/scala/kpn/core/tools/next/support/MigrateRouteRelationsTool.scala

@@ -1,8 +1,22 @@
 package kpn.core.tools.next.support
 
+import kpn.api.common.data.Node
+import kpn.api.common.data.NodeMember
+import kpn.api.common.data.RelationMember
+import kpn.api.common.data.Way
+import kpn.api.common.data.WayMember
+import kpn.api.custom.Relation
+import kpn.api.custom.Tags
+import kpn.api.custom.Timestamp
 import kpn.core.tools.next.database.NextDatabase
 import kpn.core.tools.next.database.NextDatabaseImpl
 import kpn.core.tools.next.domain.NextRouteRelation
+import kpn.core.tools.next.domain.OldNode
+import kpn.core.tools.next.domain.OldNodeMember
+import kpn.core.tools.next.domain.OldRelation
+import kpn.core.tools.next.domain.OldRelationMember
+import kpn.core.tools.next.domain.OldWay
+import kpn.core.tools.next.domain.OldWayMember
 import kpn.database.util.Mongo.codecRegistry
 import org.mongodb.scala.MongoClient
 
@@ -30,16 +44,72 @@ class MigrateRouteRelationsTool(database: NextDatabase) {
  println(s"${index + 1}/${ids.size}")
  }
  database.oldRouteRelations.findById(id).foreach { oldDocument =>
+ val relation = toRelation(oldDocument.relation)
  val newDocument = NextRouteRelation(
- oldDocument._id,
- oldDocument.relation.raw.version,
- oldDocument.relation.raw.timestamp,
- oldDocument.relation.raw.changeSetId,
- oldDocument.relation.raw.tags,
- oldDocument.relation.members
+ relation.id,
+ relation.version,
+ relation.timestamp,
+ relation.changeSetId,
+ relation.tags,
+ relation.members
  )
  database.routeRelations.save(newDocument)
  }
  }
  }
+
+ private def toNode(oldNode: OldNode): Node = {
+ Node(
+ oldNode.raw.id,
+ oldNode.raw.latitude,
+ oldNode.raw.longitude,
+ oldNode.raw.version,
+ oldNode.raw.timestamp,
+ oldNode.raw.changeSetId,
+ oldNode.raw.tags
+ )
+ }
+
+ private def toWay(oldWay: OldWay): Way = {
+ Way(
+ oldWay.raw.id,
+ oldWay.raw.version,
+ oldWay.raw.timestamp,
+ oldWay.raw.changeSetId,
+ oldWay.raw.tags,
+ oldWay.nodes.map(toNode).toVector,
+ oldWay.length
+ )
+ }
+
+ private def toRelation(oldRelation: OldRelation): Relation = {
+
+ val newMembers = oldRelation.members.map {
+ case nodeMember: OldNodeMember =>
+ NodeMember(
+ toNode(nodeMember.node),
+ nodeMember.role
+ )
+ case wayMember: OldWayMember =>
+ WayMember(
+ toWay(wayMember.way),
+ wayMember.role
+ )
+
+ case relationMember: OldRelationMember =>
+ RelationMember(
+ toRelation(relationMember.relation),
+ relationMember.role
+ )
+ }
+
+ Relation(
+ oldRelation.raw.id: Long,
+ oldRelation.raw.version: Long,
+ oldRelation.raw.timestamp: Timestamp,
+ oldRelation.raw.changeSetId: Long,
+ oldRelation.raw.tags: Tags,
+ newMembers
+ )
+ }
 }

server/src/main/scala/kpn/core/tools/next/support/NextCreateRouteRelationsTool.scala

@@ -25,7 +25,7 @@ object NextCreateRouteRelationsTool {
  val database = new NextDatabaseImpl(mongoDatabase)
  val overpassQueryExecutor = new OverpassQueryExecutorRemoteImpl()
  val tool = new NextCreateRouteRelationsTool(database, overpassQueryExecutor)
- tool.createMonitorRelations()
+ tool.createRouteRelations()
  } finally {
  client.close()
  }
@@ -37,24 +37,24 @@ class NextCreateRouteRelationsTool(database: NextDatabase, overpassQueryExecutor
  private val log = Log(classOf[NextCreateRouteRelationsTool])
  private val batchSize = 100
 
- def createMonitorRelations(): Unit = {
+ def createRouteRelations(): Unit = {
  val routeIds = collectRouteIds()
  log.info(s"collected ${routeIds.size} route ids")
  routeIds.sliding(batchSize, batchSize).zipWithIndex.foreach { case (batchRouteIds, index) =>
  log.info(s"${index * batchSize}/${routeIds.size}")
- createMonitorRelations(batchRouteIds)
+ createRouteRelationBatch(batchRouteIds)
  }
  }
 
  private def collectRouteIds(): Seq[Long] = {
  val file = new File(Dirs.root, "next/route-ids.txt")
  val allRouteIds = FileUtils.readFileToString(file, "UTF-8").split("\n").map(_.toLong)
- val loadedRouteIds = database.routeRelations.stringIds().map(_.toLong)
+ val loadedRouteIds = database.routeRelations.ids()
  (allRouteIds.toSet -- loadedRouteIds.toSet).toSeq.sorted
  }
 
- private def createMonitorRelations(routeRelationIds: Seq[Long]): Unit = {
- val data = monitorRelationData(routeRelationIds)
+ private def createRouteRelationBatch(routeRelationIds: Seq[Long]): Unit = {
+ val data = overpassRelationData(routeRelationIds)
  routeRelationIds.foreach { routeRelationId =>
  data.relations.get(routeRelationId) match {
  case Some(relation) =>
@@ -76,7 +76,7 @@ class NextCreateRouteRelationsTool(database: NextDatabase, overpassQueryExecutor
  }
  }
 
- private def monitorRelationData(relationIds: Seq[Long]): Data = {
+ private def overpassRelationData(relationIds: Seq[Long]): Data = {
  val relations = relationIds.map(id => s"relation($id);").mkString
  val query = s"($relations);(._;>;);out meta;"
  val xmlString = overpassQueryExecutor.execute(query)

server/src/main/scala/kpn/core/tools/next/support/TryoutReadReplicateTool.scala

@@ -9,13 +9,13 @@ import org.apache.commons.io.IOUtils
 import java.nio.charset.StandardCharsets
 import java.util.zip.GZIPInputStream
 
-object MonitorReadReplicateTool {
+object TryoutReadReplicateTool {
  def main(args: Array[String]): Unit = {
- new MonitorReadReplicateTool().tryout()
+ new TryoutReadReplicateTool().tryout()
  }
 }
 
-class MonitorReadReplicateTool {
+class TryoutReadReplicateTool {
  def tryout(): Unit = {
  val session = openSession()
  try {

server/src/main/scala/kpn/server/json/OldRelationJsonDeserializer.scala

@@ -5,14 +5,13 @@ import com.fasterxml.jackson.databind.DeserializationContext
 import com.fasterxml.jackson.databind.JsonDeserializer
 import com.fasterxml.jackson.databind.JsonMappingException
 import com.fasterxml.jackson.databind.JsonNode
-import kpn.api.common.data.Node
-import kpn.api.common.data.NodeMember
-import kpn.api.common.data.RelationMember
-import kpn.api.common.data.Way
-import kpn.api.common.data.WayMember
 import kpn.api.common.data.raw.RawRelation
-import kpn.api.custom.Relation
+import kpn.core.tools.next.domain.OldNode
+import kpn.core.tools.next.domain.OldNodeMember
 import kpn.core.tools.next.domain.OldRelation
+import kpn.core.tools.next.domain.OldRelationMember
+import kpn.core.tools.next.domain.OldWay
+import kpn.core.tools.next.domain.OldWayMember
 
 import scala.jdk.CollectionConverters.IteratorHasAsScala
 
@@ -24,16 +23,16 @@ class OldRelationJsonDeserializer extends JsonDeserializer[OldRelation] {
  val members = node.get("members").iterator().asScala.toSeq.map { member =>
  val role = Json.objectMapper.treeToValue(member.get("role"), classOf[Option[String]])
  if (member.has("way")) {
- val way = Json.objectMapper.treeToValue(member.get("way"), classOf[Way])
- WayMember(way, role)
+ val way = Json.objectMapper.treeToValue(member.get("way"), classOf[OldWay])
+ OldWayMember(way, role)
  }
  else if (member.has("node")) {
- val node = Json.objectMapper.treeToValue(member.get("node"), classOf[Node])
- NodeMember(node, role)
+ val node = Json.objectMapper.treeToValue(member.get("node"), classOf[OldNode])
+ OldNodeMember(node, role)
  }
  else if (member.has("relation")) {
- val memberRelation = Json.objectMapper.treeToValue(member.get("relation"), classOf[Relation])
- RelationMember(memberRelation, role)
+ val memberRelation = Json.objectMapper.treeToValue(member.get("relation"), classOf[OldRelation])
+ OldRelationMember(memberRelation, role)
  }
  else {
  throw JsonMappingException.from(