Merge pull request #557 from NDLANO/migrate-resources-urls

Add migration to convert resource paths
NDLANO · Dec 9, 2024 · 397ef07 · 397ef07
2 parents 1bf76f5 + 8235ec9
commit 397ef07
Show file tree

Hide file tree

Showing 6 changed files with 126 additions and 34 deletions.
diff --git a/database/src/main/scala/no/ndla/database/TableMigration.scala b/database/src/main/scala/no/ndla/database/TableMigration.scala
@@ -9,11 +9,12 @@
 package no.ndla.database
 
 import org.flywaydb.core.api.migration.{BaseJavaMigration, Context}
-import scalikejdbc.{DB, DBSession, *}
+import scalikejdbc.*
 
 abstract class TableMigration[ROW_DATA] extends BaseJavaMigration {
   val tableName: String
   val whereClause: SQLSyntax
+  val chunkSize: Int = 1000
   def extractRowData(rs: WrappedResultSet): ROW_DATA
   def updateRow(rowData: ROW_DATA)(implicit session: DBSession): Int
   lazy val tableNameSQL: SQLSyntax = SQLSyntax.createUnsafely(tableName)
@@ -25,7 +26,7 @@ abstract class TableMigration[ROW_DATA] extends BaseJavaMigration {
   }
 
   private def allRows(offset: Long)(implicit session: DBSession): Seq[ROW_DATA] = {
-    sql"select * from $tableNameSQL where $whereClause order by id limit 1000 offset $offset"
+    sql"select * from $tableNameSQL where $whereClause order by id limit $chunkSize offset $offset"
       .map(rs => extractRowData(rs))
       .list()
   }
@@ -36,11 +37,11 @@ abstract class TableMigration[ROW_DATA] extends BaseJavaMigration {
 
   private def migrateRows(implicit session: DBSession): Unit = {
     val count        = countAllRows.get
-    var numPagesLeft = (count / 1000) + 1
+    var numPagesLeft = (count / chunkSize) + 1
     var offset       = 0L
 
     while (numPagesLeft > 0) {
-      allRows(offset * 1000).map { rowData => updateRow(rowData) }: Unit
+      allRows(offset * chunkSize).map { rowData => updateRow(rowData) }: Unit
       numPagesLeft -= 1
       offset += 1
     }

diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala
@@ -21,7 +21,8 @@ import no.ndla.myndlaapi.controller.{
   SwaggerDocControllerConfig,
   UserController
 }
-import no.ndla.myndlaapi.integration.SearchApiClient
+import no.ndla.myndlaapi.db.migrationwithdependencies.V16__MigrateResourcePaths
+import no.ndla.myndlaapi.integration.{SearchApiClient, TaxonomyApiClient}
 import no.ndla.myndlaapi.integration.nodebb.NodeBBClient
 import no.ndla.myndlaapi.repository.{ArenaRepository, ConfigRepository, FolderRepository, UserRepository}
 import no.ndla.myndlaapi.service.{
@@ -70,37 +71,41 @@ class ComponentRegistry(properties: MyNdlaApiProperties)
     with NodeBBClient
     with InternController
     with SearchApiClient
+    with TaxonomyApiClient
+    with V16__MigrateResourcePaths
     with NdlaClient {
   override val props: MyNdlaApiProperties = properties
 
-  lazy val healthController: TapirHealthController        = new TapirHealthController
-  lazy val clock: SystemClock                             = new SystemClock
-  lazy val migrator: DBMigrator                           = DBMigrator()
-  lazy val folderController: FolderController             = new FolderController
-  lazy val feideApiClient: FeideApiClient                 = new FeideApiClient
-  lazy val redisClient                                    = new RedisClient(props.RedisHost, props.RedisPort)
-  lazy val folderRepository: FolderRepository             = new FolderRepository
-  lazy val folderConverterService: FolderConverterService = new FolderConverterService
-  lazy val folderReadService: FolderReadService           = new FolderReadService
-  lazy val folderWriteService: FolderWriteService         = new FolderWriteService
-  lazy val userRepository: UserRepository                 = new UserRepository
-  lazy val userService: UserService                       = new UserService
-  lazy val userController: UserController                 = new UserController
-  lazy val configRepository: ConfigRepository             = new ConfigRepository
-  lazy val configService: ConfigService                   = new ConfigService
-  lazy val configController: ConfigController             = new ConfigController
-  lazy val statsController: StatsController               = new StatsController
-  lazy val arenaRepository: ArenaRepository               = new ArenaRepository
-  lazy val arenaReadService: ArenaReadService             = new ArenaReadService
-  lazy val arenaController: ArenaController               = new ArenaController
-  lazy val converterService: ConverterService             = new ConverterService
-  lazy val importService: ImportService                   = new ImportService
-  lazy val nodebb: NodeBBClient                           = new NodeBBClient
-  lazy val internController: InternController             = new InternController
-  lazy val searchApiClient: SearchApiClient               = new SearchApiClient
-  lazy val ndlaClient: NdlaClient                         = new NdlaClient
-  lazy val myndlaApiClient: MyNDLAApiClient               = new MyNDLAApiClient
+  lazy val healthController: TapirHealthController              = new TapirHealthController
+  lazy val clock: SystemClock                                   = new SystemClock
+  lazy val folderController: FolderController                   = new FolderController
+  lazy val feideApiClient: FeideApiClient                       = new FeideApiClient
+  lazy val redisClient                                          = new RedisClient(props.RedisHost, props.RedisPort)
+  lazy val folderRepository: FolderRepository                   = new FolderRepository
+  lazy val folderConverterService: FolderConverterService       = new FolderConverterService
+  lazy val folderReadService: FolderReadService                 = new FolderReadService
+  lazy val folderWriteService: FolderWriteService               = new FolderWriteService
+  lazy val userRepository: UserRepository                       = new UserRepository
+  lazy val userService: UserService                             = new UserService
+  lazy val userController: UserController                       = new UserController
+  lazy val configRepository: ConfigRepository                   = new ConfigRepository
+  lazy val configService: ConfigService                         = new ConfigService
+  lazy val configController: ConfigController                   = new ConfigController
+  lazy val statsController: StatsController                     = new StatsController
+  lazy val arenaRepository: ArenaRepository                     = new ArenaRepository
+  lazy val arenaReadService: ArenaReadService                   = new ArenaReadService
+  lazy val arenaController: ArenaController                     = new ArenaController
+  lazy val converterService: ConverterService                   = new ConverterService
+  lazy val importService: ImportService                         = new ImportService
+  lazy val nodebb: NodeBBClient                                 = new NodeBBClient
+  lazy val internController: InternController                   = new InternController
+  lazy val searchApiClient: SearchApiClient                     = new SearchApiClient
+  lazy val taxonomyApiClient: TaxonomyApiClient                 = new TaxonomyApiClient
+  lazy val ndlaClient: NdlaClient                               = new NdlaClient
+  lazy val myndlaApiClient: MyNDLAApiClient                     = new MyNDLAApiClient
+  lazy val v16__MigrateResourcePaths: V16__MigrateResourcePaths = new V16__MigrateResourcePaths
 
+  override val migrator: DBMigrator         = DBMigrator(v16__MigrateResourcePaths)
   override val dataSource: HikariDataSource = DataSource.getHikariDataSource
   DataSource.connectToDatabase()
 

diff --git a/...main/scala/no/ndla/myndlaapi/db/migrationwithdependencies/V16__MigrateResourcePaths.scala b/...main/scala/no/ndla/myndlaapi/db/migrationwithdependencies/V16__MigrateResourcePaths.scala
@@ -0,0 +1,42 @@
+/*
+ * Part of NDLA myndla-api
+ * Copyright (C) 2024 NDLA
+ *
+ * See LICENSE
+ */
+
+package no.ndla.myndlaapi.db.migrationwithdependencies
+
+import no.ndla.database.TableMigration
+import no.ndla.myndlaapi.integration.TaxonomyApiClient
+import no.ndla.network.NdlaClient
+import scalikejdbc.{DBSession, WrappedResultSet, scalikejdbcSQLInterpolationImplicitDef}
+
+import java.util.UUID
+
+trait V16__MigrateResourcePaths {
+  this: TaxonomyApiClient & NdlaClient =>
+
+  class V16__MigrateResourcePaths extends TableMigration[ResourceRow] {
+    override val tableName: String                  = "resources"
+    override val whereClause: scalikejdbc.SQLSyntax = sqls"path is not null"
+    override val chunkSize: Int                     = 1000
+    override def extractRowData(rs: WrappedResultSet): ResourceRow = ResourceRow(
+      UUID.fromString(rs.string("id")),
+      rs.string("resource_type"),
+      rs.string("path")
+    )
+    override def updateRow(rowData: ResourceRow)(implicit session: DBSession): Int = {
+      rowData.resourceType match {
+        case "article" | "learningpath" | "multidisciplinary" | "topic" =>
+          taxonomyApiClient
+            .resolveUrl(rowData.path)
+            .map { path => sql"update resources set path=$path where id = ${rowData.id}".update() }
+            .get
+        case _ => 0
+      }
+    }
+  }
+}
+
+case class ResourceRow(id: UUID, resourceType: String, path: String)
diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/SearchApiClient.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/SearchApiClient.scala
@@ -17,7 +17,7 @@ import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
 
 trait SearchApiClient {
-  this: NdlaClient with Props =>
+  this: NdlaClient & Props =>
 
   val searchApiClient: SearchApiClient
 

diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/TaxonomyApiClient.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/TaxonomyApiClient.scala
@@ -0,0 +1,38 @@
+/*
+ * Part of NDLA myndla-api
+ * Copyright (C) 2024 NDLA
+ *
+ * See LICENSE
+ */
+
+package no.ndla.myndlaapi.integration
+
+import com.typesafe.scalalogging.StrictLogging
+import io.circe.{Decoder, Encoder}
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import no.ndla.myndlaapi.Props
+import no.ndla.network.NdlaClient
+import sttp.client3.quick.*
+
+import scala.util.{Success, Try}
+
+trait TaxonomyApiClient {
+  this: NdlaClient & Props =>
+
+  val taxonomyApiClient: TaxonomyApiClient
+
+  class TaxonomyApiClient extends StrictLogging {
+    private val resolveEndpoint = s"${props.TaxonomyUrl}/v1/url/resolve"
+
+    def resolveUrl(path: String): Try[String] = {
+      val req = quickRequest.get(uri"$resolveEndpoint?path=$path")
+      ndlaClient.fetch[ResolvePathResponse](req).map(resolved => resolved.url).orElse(Success(path))
+    }
+  }
+}
+
+case class ResolvePathResponse(url: String)
+object ResolvePathResponse {
+  implicit def decoder: Decoder[ResolvePathResponse] = deriveDecoder
+  implicit def encoder: Encoder[ResolvePathResponse] = deriveEncoder
+}
diff --git a/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala b/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala
@@ -116,7 +116,13 @@ class FolderRepositoryTest
     val resource2 =
       repository.insertResource("feide", "/path2", ResourceType.Topic, created, TestData.baseResourceDocument)
     val resource3 =
-      repository.insertResource("feide", "/path3", ResourceType.Multidisciplinary, created, TestData.baseResourceDocument)
+      repository.insertResource(
+        "feide",
+        "/path3",
+        ResourceType.Multidisciplinary,
+        created,
+        TestData.baseResourceDocument
+      )
     val resource4 =
       repository.insertResource("feide", "/path4", ResourceType.Image, created, TestData.baseResourceDocument)
     val resource5 =