From 3ba8e06c86333175cc60f90e27b5b353958361bd Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Thu, 12 Dec 2024 15:11:22 +0100 Subject: [PATCH 1/8] search-api: Use grep dump to fetch `GrepBundle` This downloads and extracts a zip of the grep dump to build the `GrepBundle`. This allows us to extract more information about each kode, doesn't really seem that much slower either. --- .../searchapi/integration/GrepApiClient.scala | 151 ++++++++++-------- .../ndla/searchapi/integration/ZipUtil.scala | 34 ++++ .../searchapi/model/grep/GrepElement.scala | 36 +++-- .../search/SearchConverterService.scala | 13 +- .../scala/no/ndla/searchapi/TestData.scala | 19 ++- .../integration/GrepApiClientTest.scala | 21 +++ .../search/GrepSearchServiceTest.scala | 23 ++- .../search/SearchConverterServiceTest.scala | 19 ++- 8 files changed, 218 insertions(+), 98 deletions(-) create mode 100644 search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala create mode 100644 search-api/src/test/scala/no/ndla/searchapi/integration/GrepApiClientTest.scala diff --git a/search-api/src/main/scala/no/ndla/searchapi/integration/GrepApiClient.scala b/search-api/src/main/scala/no/ndla/searchapi/integration/GrepApiClient.scala index 07d6202b3..92c161064 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/integration/GrepApiClient.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/integration/GrepApiClient.scala @@ -7,20 +7,22 @@ package no.ndla.searchapi.integration -import java.util.concurrent.Executors +import cats.implicits.toTraverseOps import com.typesafe.scalalogging.StrictLogging import io.circe.Decoder +import no.ndla.common.CirceUtil +import no.ndla.common.implicits.TryQuestionMark +import no.ndla.common.model.NDLADate import no.ndla.network.NdlaClient -import no.ndla.network.model.RequestInfo import no.ndla.searchapi.Props import no.ndla.searchapi.caching.Memoize -import no.ndla.searchapi.model.api.GrepException import no.ndla.searchapi.model.grep.* import sttp.client3.quick.* -import scala.concurrent.duration.{Duration, DurationInt} -import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutor, Future} -import scala.util.{Failure, Success, Try} +import java.io.File +import java.nio.file.Files +import scala.util.Using.Releasable +import scala.util.{Failure, Success, Try, Using} trait GrepApiClient { this: NdlaClient & Props => @@ -28,75 +30,96 @@ trait GrepApiClient { class GrepApiClient extends StrictLogging { import props.GrepApiUrl - private val GrepApiEndpoint = s"$GrepApiUrl/kl06/v201906" + private val grepDumpUrl = s"$GrepApiUrl/kl06/v201906/dump/json" - private def getAllKjerneelementer: Try[List[GrepKjerneelement]] = - get[List[GrepKjerneelement]](s"$GrepApiEndpoint/kjerneelementer-lk20/").map(_.distinct) - - private def getAllKompetansemaal: Try[List[GrepKompetansemaal]] = - get[List[GrepKompetansemaal]](s"$GrepApiEndpoint/kompetansemaal-lk20/").map(_.distinct) - - private def getAllKompetansemaalSett: Try[List[GrepKompetansemaalSett]] = - get[List[GrepKompetansemaalSett]](s"$GrepApiEndpoint/kompetansemaalsett-lk20/").map(_.distinct) - - private def getAllTverrfagligeTemaer: Try[List[GrepTverrfagligTema]] = - get[List[GrepTverrfagligTema]](s"$GrepApiEndpoint/tverrfaglige-temaer-lk20/").map(_.distinct) - - private def getAllLaereplaner: Try[List[GrepLaererplan]] = - get[List[GrepLaererplan]](s"$GrepApiEndpoint/laereplaner-lk20/").map(_.distinct) + private def readFile(file: File): Try[String] = Try { + Using.resource(scala.io.Source.fromFile(file)) { source => + source.getLines().mkString + } + } - // NOTE: We add a helper so we don't have to provide `()` where this is used :^) - val getGrepBundle: () => Try[GrepBundle] = () => _getGrepBundle(()) + private def readGrepJsonFiles[T](dump: File, path: String)(implicit d: Decoder[T]): Try[List[T]] = { + val folder = new File(dump, path) + val jsonFiles = folder.list() + jsonFiles.toList.traverse { f => + for { + jsonStr <- readFile(new File(folder, f)) + parsed <- CirceUtil.tryParseAs[T](jsonStr) + } yield parsed + } + } + private def getKjerneelementerLK20(dump: File): Try[List[GrepKjerneelement]] = + readGrepJsonFiles[GrepKjerneelement](dump, "kjerneelementer-lk20") + private def getKompetansemaalLK20(dump: File): Try[List[GrepKompetansemaal]] = + readGrepJsonFiles[GrepKompetansemaal](dump, "kompetansemaal-lk20") + private def getKompetansemaalsettLK20(dump: File): Try[List[GrepKompetansemaalSett]] = + readGrepJsonFiles[GrepKompetansemaalSett](dump, "kompetansemaalsett-lk20") + private def getTverrfagligeTemaerLK20(dump: File): Try[List[GrepTverrfagligTema]] = + readGrepJsonFiles[GrepTverrfagligTema](dump, "tverrfaglige-temaer-lk20") + private def getLaereplanerLK20(dump: File): Try[List[GrepLaererplan]] = + readGrepJsonFiles[GrepLaererplan](dump, "laereplaner-lk20") + + private def getBundleFromDump(dump: File): Try[GrepBundle] = for { + kjerneelementer <- getKjerneelementerLK20(dump) + kompetansemaal <- getKompetansemaalLK20(dump) + kompetansemaalsett <- getKompetansemaalsettLK20(dump) + tverrfagligeTemaer <- getTverrfagligeTemaerLK20(dump) + laereplaner <- getLaereplanerLK20(dump) + } yield GrepBundle( + kjerneelementer = kjerneelementer, + kompetansemaal = kompetansemaal, + kompetansemaalsett = kompetansemaalsett, + tverrfagligeTemaer = tverrfagligeTemaer, + laereplaner = laereplaner + ) + + val getGrepBundle: () => Try[GrepBundle] = () => _getGrepBundle(()) private val _getGrepBundle: Memoize[Unit, Try[GrepBundle]] = new Memoize(1000 * 60, _ => getGrepBundleUncached) - /** The memoized function of this [[getGrepBundle]] should probably be used in most cases */ - private def getGrepBundleUncached: Try[GrepBundle] = { - logger.info("Fetching grep in bulk...") - val startFetch = System.currentTimeMillis() - implicit val ec: ExecutionContextExecutor = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(3)) - - val requestInfo = RequestInfo.fromThreadContext() - - /** Calls function in separate thread and converts Try to Future */ - def tryToFuture[T](x: () => Try[T]) = Future { - requestInfo.setThreadContextRequestInfo() - x() - }.flatMap(Future.fromTry) + implicit object FileIsReleasable extends Releasable[File] { + private def deleteDirectory(f: File): Unit = { + if (f.isDirectory) { + f.listFiles().foreach(deleteDirectory) + } + f.delete(): Unit + } + def release(resource: File): Unit = deleteDirectory(resource) + } - val kjerneelementer = tryToFuture(() => getAllKjerneelementer) - val kompetansemaal = tryToFuture(() => getAllKompetansemaal) - val kompetansemaalsett = tryToFuture(() => getAllKompetansemaalSett) - val tverrfagligeTemaer = tryToFuture(() => getAllTverrfagligeTemaer) - val laererplaner = tryToFuture(() => getAllLaereplaner) + private def getGrepBundleUncached: Try[GrepBundle] = { + val date = NDLADate.now().toUTCEpochSecond + val tempDirPath = Try(Files.createTempDirectory(s"grep-dump-$date")).? + Using(tempDirPath.toFile) { tempDir => + val zippedDump = fetchDump(tempDir).? + val unzippedDump = ZipUtil.unzip(zippedDump, tempDir, deleteArchive = true).? + val bundle = getBundleFromDump(unzippedDump).? + logger.info("Successfully fetched grep bundle") + bundle + } + } - val x = for { - kjerne <- kjerneelementer - kompetanse <- kompetansemaal - kompetansesett <- kompetansemaalsett - tverrfag <- tverrfagligeTemaer - laere <- laererplaner - } yield GrepBundle( - kjerneelementer = kjerne, - kompetansemaal = kompetanse, - kompetansemaalsett = kompetansesett, - tverrfagligeTemaer = tverrfag, - laereplaner = laere - ) + case class GrepDumpDownloadException(message: String) extends RuntimeException(message) { + def withCause(cause: Throwable): GrepDumpDownloadException = { + initCause(cause) + this + } + } - Try(Await.result(x, Duration(300, "seconds"))) match { - case Success(bundle) => - logger.info(s"Fetched grep in ${System.currentTimeMillis() - startFetch}ms...") - Success(bundle) + private def fetchDump(tempDir: File): Try[File] = { + val outputFile = new File(tempDir, "grep-dump.zip") + logger.info(s"Downloading grep dump from $grepDumpUrl to ${outputFile.getAbsolutePath}") + val request = quickRequest + .get(uri"$grepDumpUrl") + .response(asFile(outputFile)) + Try(simpleHttpClient.send(request)) match { + case Success(response) if response.isSuccess => Success(outputFile) + case Success(response) => + Failure(GrepDumpDownloadException(s"Failed to fetch grep dump: ${response.statusText}")) case Failure(ex) => - logger.error(s"Could not fetch grep bundle (${ex.getMessage})", ex) - Failure(GrepException("Could not fetch grep bundle...")) + Failure(GrepDumpDownloadException(s"Failed to fetch grep dump: ${ex.getMessage}").withCause(ex)) } } - private def get[A: Decoder](url: String, params: (String, String)*): Try[A] = { - val request = quickRequest.get(uri"$url?$params").readTimeout(60.seconds) - ndlaClient.fetch[A](request) - } } } diff --git a/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala b/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala new file mode 100644 index 000000000..767f93909 --- /dev/null +++ b/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala @@ -0,0 +1,34 @@ +/* + * Part of NDLA search-api + * Copyright (C) 2024 NDLA + * + * See LICENSE + * + */ + +package no.ndla.searchapi.integration + +import java.io.* +import java.util.zip.ZipInputStream +import scala.util.Try + +object ZipUtil { + def unzip(zipFile: File, targetDir: File, deleteArchive: Boolean): Try[File] = Try { + val fis = new FileInputStream(zipFile) + val zis = new ZipInputStream(fis) + LazyList.continually(zis.getNextEntry).takeWhile(_ != null).foreach { file => + val outFile = new File(targetDir, file.getName) + outFile.getParentFile.mkdirs() + val fout = new FileOutputStream(outFile) + val buffer = new Array[Byte](1024) + LazyList + .continually(zis.read(buffer)) + .takeWhile(_ != -1) + .foreach(fout.write(buffer, 0, _)) + } + + if (deleteArchive) zipFile.delete() + + targetDir + } +} diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala index 9ea8db746..6b8dd65c9 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala @@ -12,16 +12,24 @@ import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} sealed trait GrepElement { val kode: String - val tittel: Seq[GrepTitle] + def getTitle: Seq[GrepTitle] } sealed trait BelongsToLaerePlan { - val tilhoerer_laereplan: BelongsToObj + val `tilhoerer-laereplan`: BelongsToObj } -case class GrepKjerneelement(kode: String, tittel: Seq[GrepTitle], tilhoerer_laereplan: BelongsToObj) +case class TitleObj(tekst: List[GrepTitle]) +object TitleObj { + implicit val encoder: Encoder[TitleObj] = deriveEncoder + implicit val decoder: Decoder[TitleObj] = deriveDecoder +} + +case class GrepKjerneelement(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj) extends GrepElement - with BelongsToLaerePlan + with BelongsToLaerePlan { + override def getTitle: Seq[GrepTitle] = tittel.tekst +} object GrepKjerneelement { implicit val encoder: Encoder[GrepKjerneelement] = deriveEncoder implicit val decoder: Decoder[GrepKjerneelement] = deriveDecoder @@ -33,29 +41,37 @@ object BelongsToObj { implicit val decoder: Decoder[BelongsToObj] = deriveDecoder } -case class GrepKompetansemaal(kode: String, tittel: Seq[GrepTitle], tilhoerer_laereplan: BelongsToObj) +case class GrepKompetansemaal(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj) extends GrepElement - with BelongsToLaerePlan + with BelongsToLaerePlan { + override def getTitle: Seq[GrepTitle] = tittel.tekst +} object GrepKompetansemaal { implicit val encoder: Encoder[GrepKompetansemaal] = deriveEncoder implicit val decoder: Decoder[GrepKompetansemaal] = deriveDecoder } -case class GrepKompetansemaalSett(kode: String, tittel: Seq[GrepTitle], tilhoerer_laereplan: BelongsToObj) +case class GrepKompetansemaalSett(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj) extends GrepElement - with BelongsToLaerePlan + with BelongsToLaerePlan { + override def getTitle: Seq[GrepTitle] = tittel.tekst +} object GrepKompetansemaalSett { implicit val encoder: Encoder[GrepKompetansemaalSett] = deriveEncoder implicit val decoder: Decoder[GrepKompetansemaalSett] = deriveDecoder } -case class GrepLaererplan(kode: String, tittel: Seq[GrepTitle]) extends GrepElement +case class GrepLaererplan(kode: String, tittel: TitleObj) extends GrepElement { + override def getTitle: Seq[GrepTitle] = tittel.tekst +} object GrepLaererplan { implicit val encoder: Encoder[GrepLaererplan] = deriveEncoder implicit val decoder: Decoder[GrepLaererplan] = deriveDecoder } -case class GrepTverrfagligTema(kode: String, tittel: Seq[GrepTitle]) extends GrepElement +case class GrepTverrfagligTema(kode: String, tittel: Seq[GrepTitle]) extends GrepElement { + override def getTitle: Seq[GrepTitle] = tittel +} object GrepTverrfagligTema { implicit val encoder: Encoder[GrepTverrfagligTema] = deriveEncoder implicit val decoder: Decoder[GrepTverrfagligTema] = deriveDecoder diff --git a/search-api/src/main/scala/no/ndla/searchapi/service/search/SearchConverterService.scala b/search-api/src/main/scala/no/ndla/searchapi/service/search/SearchConverterService.scala index f9e650ecf..8274eee21 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/service/search/SearchConverterService.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/service/search/SearchConverterService.scala @@ -255,8 +255,13 @@ trait SearchConverterService { } - def convertGrepTitleToLanguageValue(grepElement: GrepElement): Seq[LanguageValue[String]] = - grepElement.tittel.flatMap(gt => { + def asSearchableGrep(grepElement: GrepElement): Try[SearchableGrepElement] = { + val laererplan = grepElement match { + case lp: BelongsToLaerePlan => Some(lp.`tilhoerer-laereplan`.kode) + case _ => None + } + val defaultTitle = grepElement.getTitle.find(_.spraak == "default") + val titles = grepElement.getTitle.flatMap(gt => { ISO639.get6391CodeFor6392Code(gt.spraak) match { case Some(convertedLanguage) => Some(LanguageValue(language = convertedLanguage, value = gt.verdi.trim)) @@ -972,7 +977,9 @@ trait SearchConverterService { grepCode, grepBundle.grepContextByCode .get(grepCode) - .flatMap(element => element.tittel.find(title => title.spraak == "default").map(title => title.verdi)) + .flatMap(element => + element.getTitle.find(title => title.spraak == "default").map(title => title.verdi) + ) ) ) .toList diff --git a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala index 204e60fe1..6a2cbda2f 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala @@ -57,7 +57,8 @@ import no.ndla.searchapi.model.grep.{ GrepKompetansemaal, GrepLaererplan, GrepTitle, - GrepTverrfagligTema + GrepTverrfagligTema, + TitleObj } import no.ndla.searchapi.model.search.* import no.ndla.searchapi.model.search.settings.{MultiDraftSearchSettings, SearchSettings} @@ -1587,18 +1588,26 @@ object TestData { val grepBundle: GrepBundle = emptyGrepBundle.copy( kjerneelementer = List( - GrepKjerneelement("KE12", Seq(GrepTitle("default", "Utforsking og problemløysing")), BelongsToObj("LP1")), - GrepKjerneelement("KE34", Seq(GrepTitle("default", "Abstraksjon og generalisering")), BelongsToObj("LP1")) + GrepKjerneelement( + "KE12", + TitleObj(List(GrepTitle("default", "Utforsking og problemløysing"))), + BelongsToObj("LP1") + ), + GrepKjerneelement( + "KE34", + TitleObj(List(GrepTitle("default", "Abstraksjon og generalisering"))), + BelongsToObj("LP1") + ) ), kompetansemaal = List( GrepKompetansemaal( "KM123", - Seq(GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte")), + TitleObj(List(GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"))), BelongsToObj("LP1") ) ), tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "Demokrati og medborgerskap")))), - laereplaner = List(GrepLaererplan("LP1", Seq(GrepTitle("default", "Læreplan i norsk (NOR01-04)")))) + laereplaner = List(GrepLaererplan("LP1", TitleObj(List(GrepTitle("default", "Læreplan i norsk (NOR01-04)"))))) ) val searchSettings: SearchSettings = SearchSettings( diff --git a/search-api/src/test/scala/no/ndla/searchapi/integration/GrepApiClientTest.scala b/search-api/src/test/scala/no/ndla/searchapi/integration/GrepApiClientTest.scala new file mode 100644 index 000000000..06428bb59 --- /dev/null +++ b/search-api/src/test/scala/no/ndla/searchapi/integration/GrepApiClientTest.scala @@ -0,0 +1,21 @@ +/* + * Part of NDLA backend.search-api.test + * Copyright (C) 2024 NDLA + * + * See LICENSE + * + */ + +package no.ndla.searchapi.integration + +import no.ndla.searchapi.{TestEnvironment, UnitSuite} + +class GrepApiClientTest extends UnitSuite with TestEnvironment { + + test("do stuff") { + + val x = ??? + + } + +} diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala index e81d40e02..ad6e6d072 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala @@ -20,7 +20,8 @@ import no.ndla.searchapi.model.grep.{ GrepKompetansemaal, GrepLaererplan, GrepTitle, - GrepTverrfagligTema + GrepTverrfagligTema, + TitleObj } class GrepSearchServiceTest extends IntegrationSuite(EnableElasticsearchContainer = true) with TestEnvironment { @@ -49,21 +50,27 @@ class GrepSearchServiceTest extends IntegrationSuite(EnableElasticsearchContaine kjerneelementer = List( GrepKjerneelement( "KE12", - Seq(GrepTitle("default", "Utforsking og problemløysing"), GrepTitle("nob", "Utforsking og problemløsning")), + TitleObj( + List(GrepTitle("default", "Utforsking og problemløysing"), GrepTitle("nob", "Utforsking og problemløsning")) + ), BelongsToObj("LP1") ), GrepKjerneelement( "KE34", - Seq(GrepTitle("default", "Abstraksjon og generalisering"), GrepTitle("nob", "Abstraksjon og generalisering")), + TitleObj( + List(GrepTitle("default", "Abstraksjon og generalisering"), GrepTitle("nob", "Abstraksjon og generalisering")) + ), BelongsToObj("LP2") ) ), kompetansemaal = List( GrepKompetansemaal( "KM123", - Seq( - GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"), - GrepTitle("nob", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte") + TitleObj( + List( + GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"), + GrepTitle("nob", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte") + ) ), BelongsToObj("LP2") ) @@ -78,11 +85,11 @@ class GrepSearchServiceTest extends IntegrationSuite(EnableElasticsearchContaine laereplaner = List( GrepLaererplan( "LP1", - Seq(GrepTitle("default", "Læreplan i norsk"), GrepTitle("nob", "Læreplan i norsk")) + TitleObj(List(GrepTitle("default", "Læreplan i norsk"), GrepTitle("nob", "Læreplan i norsk"))) ), GrepLaererplan( "LP2", - Seq(GrepTitle("default", "Læreplan i engelsk"), GrepTitle("nob", "Læreplan i engelsk")) + TitleObj(List(GrepTitle("default", "Læreplan i engelsk"), GrepTitle("nob", "Læreplan i engelsk"))) ) ) ) diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala index 20d0b74c4..6b3d8446b 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala @@ -18,7 +18,8 @@ import no.ndla.searchapi.model.grep.{ GrepKjerneelement, GrepKompetansemaal, GrepTitle, - GrepTverrfagligTema + GrepTverrfagligTema, + TitleObj } import no.ndla.searchapi.model.search.{SearchTrait, SearchableArticle, SearchableGrepContext} import no.ndla.searchapi.model.taxonomy.* @@ -496,10 +497,11 @@ class SearchConverterServiceTest extends UnitSuite with TestEnvironment { val draft = TestData.emptyDomainDraft.copy(id = Some(99), grepCodes = Seq("KE12", "KM123", "TT2")) val grepBundle = TestData.emptyGrepBundle.copy( kjerneelementer = List( - GrepKjerneelement("KE12", Seq(GrepTitle("default", "tittel12")), BelongsToObj("LP123")), - GrepKjerneelement("KE34", Seq(GrepTitle("default", "tittel34")), BelongsToObj("LP123")) + GrepKjerneelement("KE12", TitleObj(List(GrepTitle("default", "tittel12"))), BelongsToObj("LP123")), + GrepKjerneelement("KE34", TitleObj(List(GrepTitle("default", "tittel34"))), BelongsToObj("LP123")) ), - kompetansemaal = List(GrepKompetansemaal("KM123", Seq(GrepTitle("default", "tittel123")), BelongsToObj("LP123"))), + kompetansemaal = + List(GrepKompetansemaal("KM123", TitleObj(List(GrepTitle("default", "tittel123"))), BelongsToObj("LP123"))), tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2")))) ) val grepContexts = List( @@ -516,11 +518,12 @@ class SearchConverterServiceTest extends UnitSuite with TestEnvironment { val draft = TestData.emptyDomainDraft.copy(id = Some(99), grepCodes = Seq.empty) val grepBundle = TestData.emptyGrepBundle.copy( kjerneelementer = List( - GrepKjerneelement("KE12", Seq(GrepTitle("default", "tittel12")), BelongsToObj("LP123")), - GrepKjerneelement("KE34", Seq(GrepTitle("default", "tittel34")), BelongsToObj("LP123")) + GrepKjerneelement("KE12", TitleObj(List(GrepTitle("default", "tittel12"))), BelongsToObj("LP123")), + GrepKjerneelement("KE34", TitleObj(List(GrepTitle("default", "tittel34"))), BelongsToObj("LP123")) ), - kompetansemaal = List(GrepKompetansemaal("KM123", Seq(GrepTitle("default", "tittel123")), BelongsToObj("LP123"))), - tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2")))) + kompetansemaal = + List(GrepKompetansemaal("KM123", TitleObj(List(GrepTitle("default", "tittel123"))), BelongsToObj("LP123"))), + tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", List(GrepTitle("default", "tittel2")))) ) val grepContexts = List.empty From fd39860e27d32d7da3aa30e7883f5fd71764a1f3 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Tue, 17 Dec 2024 09:50:30 +0100 Subject: [PATCH 2/8] draft-api: Deprecate grep-codes search This is replaced by the grep search in search-api. The endpoint will still work for now, but the data will no longer be updated when saving a draft. --- .../draftapi/controller/DraftController.scala | 1 + .../controller/InternController.scala | 21 ++++++------------ .../ndla/draftapi/service/WriteService.scala | 1 - .../controller/InternControllerTest.scala | 22 ++++--------------- .../draftapi/service/WriteServiceTest.scala | 1 - 5 files changed, 12 insertions(+), 34 deletions(-) diff --git a/draft-api/src/main/scala/no/ndla/draftapi/controller/DraftController.scala b/draft-api/src/main/scala/no/ndla/draftapi/controller/DraftController.scala index 596ab636f..6c96adf40 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/controller/DraftController.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/controller/DraftController.scala @@ -223,6 +223,7 @@ trait DraftController { .in("grep-codes") .summary("Retrieves a list of all previously used grepCodes in articles") .description("Retrieves a list of all previously used grepCodes in articles") + .deprecated() .in(queryParam) .in(pageSize) .in(pageNo) diff --git a/draft-api/src/main/scala/no/ndla/draftapi/controller/InternController.scala b/draft-api/src/main/scala/no/ndla/draftapi/controller/InternController.scala index eb7bf6242..4b2724563 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/controller/InternController.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/controller/InternController.scala @@ -52,7 +52,7 @@ trait InternController { val internController: InternController class InternController extends TapirController with StrictLogging { - import props.{DraftSearchIndex, DraftTagSearchIndex, DraftGrepCodesSearchIndex} + import props.{DraftSearchIndex, DraftTagSearchIndex} override val prefix: EndpointInput[Unit] = "intern" override val enableSwagger = false @@ -103,8 +103,7 @@ trait InternController { ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(4)) val articleIndex = createIndexFuture(articleIndexService, numShards) val tagIndex = createIndexFuture(tagIndexService, numShards) - val grepIndex = createIndexFuture(grepCodesIndexService, numShards) - val indexResults = Future.sequence(List(articleIndex, tagIndex, grepIndex)) + val indexResults = Future.sequence(List(articleIndex, tagIndex)) Await.result(indexResults, Duration.Inf).sequence match { case Failure(ex) => @@ -126,14 +125,12 @@ trait InternController { val indexes = for { articleIndex <- Future { articleIndexService.findAllIndexes(DraftSearchIndex) } tagIndex <- Future { tagIndexService.findAllIndexes(DraftTagSearchIndex) } - grepIndex <- Future { grepCodesIndexService.findAllIndexes(DraftGrepCodesSearchIndex) } - } yield (articleIndex, tagIndex, grepIndex) + } yield (articleIndex, tagIndex) val deleteResults: Seq[Try[_]] = Await.result(indexes, Duration(10, TimeUnit.MINUTES)) match { - case (Failure(articleFail), _, _) => return articleFail.getMessage.asLeft - case (_, Failure(tagFail), _) => return tagFail.getMessage.asLeft - case (_, _, Failure(grepFail)) => return grepFail.getMessage.asLeft - case (Success(articleIndexes), Success(tagIndexes), Success(grepIndexes)) => + case (Failure(articleFail), _) => return articleFail.getMessage.asLeft + case (_, Failure(tagFail)) => return tagFail.getMessage.asLeft + case (Success(articleIndexes), Success(tagIndexes)) => val articleDeleteResults = articleIndexes.map(index => { logger.info(s"Deleting article index $index") articleIndexService.deleteIndexWithName(Option(index)) @@ -142,11 +139,7 @@ trait InternController { logger.info(s"Deleting tag index $index") tagIndexService.deleteIndexWithName(Option(index)) }) - val grepDeleteResults = grepIndexes.map(index => { - logger.info(s"Deleting grep index $index") - grepCodesIndexService.deleteIndexWithName(Option(index)) - }) - articleDeleteResults ++ tagDeleteResults ++ grepDeleteResults + articleDeleteResults ++ tagDeleteResults } val (errors, successes) = deleteResults.partition(_.isFailure) diff --git a/draft-api/src/main/scala/no/ndla/draftapi/service/WriteService.scala b/draft-api/src/main/scala/no/ndla/draftapi/service/WriteService.scala index adfa1b21b..7a7f58297 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/service/WriteService.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/service/WriteService.scala @@ -71,7 +71,6 @@ trait WriteService { searchApiClient.indexDraft(article, user)(ec): Unit articleIndexService.indexAsync(articleId, article)(ec): Unit tagIndexService.indexAsync(articleId, article)(ec): Unit - grepCodesIndexService.indexAsync(articleId, article)(ec): Unit Success(()) } diff --git a/draft-api/src/test/scala/no/ndla/draftapi/controller/InternControllerTest.scala b/draft-api/src/test/scala/no/ndla/draftapi/controller/InternControllerTest.scala index 8b2db15a5..d18feebd8 100644 --- a/draft-api/src/test/scala/no/ndla/draftapi/controller/InternControllerTest.scala +++ b/draft-api/src/test/scala/no/ndla/draftapi/controller/InternControllerTest.scala @@ -71,19 +71,15 @@ class InternControllerTest extends UnitSuite with TestEnvironment with TapirCont test("That DELETE /index removes all indexes") { reset( articleIndexService, - tagIndexService, - grepCodesIndexService + tagIndexService ) when(articleIndexService.findAllIndexes(any[String])).thenReturn(Success(List("index1", "index2"))) when(tagIndexService.findAllIndexes(any[String])).thenReturn(Success(List("index7", "index8"))) - when(grepCodesIndexService.findAllIndexes(any[String])).thenReturn(Success(List("index9", "index10"))) doReturn(Success(""), Nil: _*).when(articleIndexService).deleteIndexWithName(Some("index1")) doReturn(Success(""), Nil: _*).when(articleIndexService).deleteIndexWithName(Some("index2")) doReturn(Success(""), Nil: _*).when(tagIndexService).deleteIndexWithName(Some("index7")) doReturn(Success(""), Nil: _*).when(tagIndexService).deleteIndexWithName(Some("index8")) - doReturn(Success(""), Nil: _*).when(grepCodesIndexService).deleteIndexWithName(Some("index9")) - doReturn(Success(""), Nil: _*).when(grepCodesIndexService).deleteIndexWithName(Some("index10")) { val res = simpleHttpClient.send( @@ -91,7 +87,7 @@ class InternControllerTest extends UnitSuite with TestEnvironment with TapirCont .delete(uri"http://localhost:$serverPort/intern/index") ) res.code.code should be(200) - res.body should equal("Deleted 6 indexes") + res.body should equal("Deleted 4 indexes") } verify(articleIndexService).findAllIndexes(props.DraftSearchIndex) @@ -104,10 +100,6 @@ class InternControllerTest extends UnitSuite with TestEnvironment with TapirCont verify(tagIndexService).deleteIndexWithName(Some("index8")) verifyNoMoreInteractions(tagIndexService) - verify(grepCodesIndexService).findAllIndexes(props.DraftGrepCodesSearchIndex) - verify(grepCodesIndexService).deleteIndexWithName(Some("index9")) - verify(grepCodesIndexService).deleteIndexWithName(Some("index10")) - verifyNoMoreInteractions(grepCodesIndexService) } test("That DELETE /index fails if at least one index isn't found, and no indexes are deleted") { @@ -140,13 +132,11 @@ class InternControllerTest extends UnitSuite with TestEnvironment with TapirCont ) { reset( articleIndexService, - tagIndexService, - grepCodesIndexService + tagIndexService ) when(articleIndexService.findAllIndexes(any[String])).thenReturn(Success(List("index1", "index2"))) when(tagIndexService.findAllIndexes(any[String])).thenReturn(Success(List("index7", "index8"))) - when(grepCodesIndexService.findAllIndexes(any[String])).thenReturn(Success(List("index9", "index10"))) doReturn(Success(""), Nil: _*).when(articleIndexService).deleteIndexWithName(Some("index1")) doReturn(Failure(new RuntimeException("No index with name 'index2' exists")), Nil: _*) @@ -154,8 +144,6 @@ class InternControllerTest extends UnitSuite with TestEnvironment with TapirCont .deleteIndexWithName(Some("index2")) doReturn(Success(""), Nil: _*).when(tagIndexService).deleteIndexWithName(Some("index7")) doReturn(Success(""), Nil: _*).when(tagIndexService).deleteIndexWithName(Some("index8")) - doReturn(Success(""), Nil: _*).when(grepCodesIndexService).deleteIndexWithName(Some("index9")) - doReturn(Success(""), Nil: _*).when(grepCodesIndexService).deleteIndexWithName(Some("index10")) { val res = simpleHttpClient.send( @@ -164,7 +152,7 @@ class InternControllerTest extends UnitSuite with TestEnvironment with TapirCont ) res.code.code should be(500) res.body should equal( - "Failed to delete 1 index: No index with name 'index2' exists. 5 indexes were deleted successfully." + "Failed to delete 1 index: No index with name 'index2' exists. 3 indexes were deleted successfully." ) } @@ -172,7 +160,5 @@ class InternControllerTest extends UnitSuite with TestEnvironment with TapirCont verify(articleIndexService).deleteIndexWithName(Some("index2")) verify(tagIndexService).deleteIndexWithName(Some("index7")) verify(tagIndexService).deleteIndexWithName(Some("index8")) - verify(grepCodesIndexService).deleteIndexWithName(Some("index9")) - verify(grepCodesIndexService).deleteIndexWithName(Some("index10")) } } diff --git a/draft-api/src/test/scala/no/ndla/draftapi/service/WriteServiceTest.scala b/draft-api/src/test/scala/no/ndla/draftapi/service/WriteServiceTest.scala index 1cf40c20c..09c710ae7 100644 --- a/draft-api/src/test/scala/no/ndla/draftapi/service/WriteServiceTest.scala +++ b/draft-api/src/test/scala/no/ndla/draftapi/service/WriteServiceTest.scala @@ -130,7 +130,6 @@ class WriteServiceTest extends UnitSuite with TestEnvironment { verify(draftRepository, times(0)).updateArticle(any[Draft], any[Boolean])(any) verify(articleIndexService, times(1)).indexAsync(any, any)(any) verify(tagIndexService, times(1)).indexAsync(any, any)(any) - verify(grepCodesIndexService, times(1)).indexAsync(any, any)(any) } test("That updateArticle updates only content properly") { From a61e8e3a9252005d8032916b2bc3ff405cb542f0 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Thu, 2 Jan 2025 15:36:02 +0100 Subject: [PATCH 3/8] search-api: Include more data in the grep search To allow us to completely skip calling the udir grep api from the frontend/graphql we need more data! This patch includes everything(?) we need to stop calling the grep api directly from elsewhere. --- .../main/scala/no/ndla/common/CirceUtil.scala | 9 +- .../ndla/searchapi/integration/ZipUtil.scala | 5 + .../searchapi/model/api/DescriptionDTO.scala | 27 +++ .../model/api/grep/GrepResultDTO.scala | 171 +++++++++++++++++- .../searchapi/model/grep/GrepElement.scala | 84 +++++++-- .../ndla/searchapi/model/grep/GrepTitle.scala | 18 +- .../model/search/SearchableGrepElement.scala | 4 +- .../service/search/GrepIndexService.scala | 4 +- .../service/search/GrepSearchService.scala | 19 +- .../search/SearchConverterService.scala | 23 +-- .../scala/no/ndla/searchapi/TestData.scala | 44 ++--- .../search/GrepSearchServiceTest.scala | 74 ++++---- .../search/SearchConverterServiceTest.scala | 30 +-- typescript/types-backend/search-api.ts | 58 +++++- 14 files changed, 428 insertions(+), 142 deletions(-) create mode 100644 search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala diff --git a/common/src/main/scala/no/ndla/common/CirceUtil.scala b/common/src/main/scala/no/ndla/common/CirceUtil.scala index 14ed2cd7a..9f1bdcc8c 100644 --- a/common/src/main/scala/no/ndla/common/CirceUtil.scala +++ b/common/src/main/scala/no/ndla/common/CirceUtil.scala @@ -16,9 +16,12 @@ import scala.util.{Failure, Try} object CirceUtil { // NOTE: Circe's `DecodingFailure` does not include a stack trace, so we wrap it in our own exception // to make it more like other failures. - case class CirceFailure(message: String) extends RuntimeException(message) + case class CirceFailure(message: String, jsonString: String) extends RuntimeException(message) object CirceFailure { - def apply(reason: Throwable): Throwable = new CirceFailure(reason.getMessage).initCause(reason) + def apply(jsonString: String, reason: Throwable): Throwable = { + val message = s"${reason.getMessage}\n$jsonString" + new CirceFailure(message, jsonString).initCause(reason) + } } def tryParseAs[T](str: String)(implicit d: Decoder[T]): Try[T] = { @@ -26,7 +29,7 @@ object CirceUtil { .parse(str) .toTry .flatMap(_.as[T].toTry) - .recoverWith { ex => Failure(CirceFailure(ex)) } + .recoverWith { ex => Failure(CirceFailure(str, ex)) } } /** This might throw an exception! Use with care, probably only use this in tests */ diff --git a/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala b/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala index 767f93909..1245bf8fa 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala @@ -25,8 +25,13 @@ object ZipUtil { .continually(zis.read(buffer)) .takeWhile(_ != -1) .foreach(fout.write(buffer, 0, _)) + + fout.close() } + zis.close() + fis.close() + if (deleteArchive) zipFile.delete() targetDir diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala b/search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala new file mode 100644 index 000000000..31d044fa3 --- /dev/null +++ b/search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala @@ -0,0 +1,27 @@ +/* + * Part of NDLA search-api + * Copyright (C) 2018 NDLA + * + * See LICENSE + */ + +package no.ndla.searchapi.model.api + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} +import no.ndla.language.model.LanguageField +import sttp.tapir.Schema.annotations.description + +@description("Title of resource") +case class DescriptionDTO( + @description("The freetext description of the resource") description: String, + @description("ISO 639-1 code that represents the language used in title") language: String +) extends LanguageField[String] { + override def value: String = description + override def isEmpty: Boolean = description.isEmpty +} + +object DescriptionDTO { + implicit val encoder: Encoder[DescriptionDTO] = deriveEncoder + implicit val decoder: Decoder[DescriptionDTO] = deriveDecoder +} diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala index 57ce3faea..c5eb38010 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala @@ -8,19 +8,170 @@ package no.ndla.searchapi.model.api.grep -import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} -import io.circe.{Decoder, Encoder} -import no.ndla.searchapi.model.api.TitleDTO +import cats.implicits.* +import io.circe.generic.auto.* +import sttp.tapir.generic.auto.* +import io.circe.syntax.* +import io.circe.{Decoder, Encoder, Json} +import no.ndla.language.Language +import no.ndla.language.Language.findByLanguageOrBestEffort +import no.ndla.search.model.LanguageValue +import no.ndla.searchapi.model.api.{DescriptionDTO, TitleDTO} +import no.ndla.searchapi.model.grep.{ + GrepKjerneelement, + GrepKompetansemaal, + GrepKompetansemaalSett, + GrepLaererplan, + GrepTitle, + GrepTverrfagligTema +} +import no.ndla.searchapi.model.search.SearchableGrepElement +import sttp.tapir.Schema import sttp.tapir.Schema.annotations.description +import scala.util.{Success, Try} + @description("Information about a single grep search result entry") -case class GrepResultDTO( - @description("The grep code") code: String, - @description("The greps title") title: TitleDTO, - @description("The grep laereplan") laereplanCode: Option[String] -) +sealed trait GrepResultDTO { + @description("The grep code") val code: String + @description("The greps title") val title: TitleDTO +} object GrepResultDTO { - implicit val encoder: Encoder[GrepResultDTO] = deriveEncoder - implicit val decoder: Decoder[GrepResultDTO] = deriveDecoder + implicit val encoder: Encoder[GrepResultDTO] = Encoder.instance[GrepResultDTO] { result => + val json = result match { + case x: GrepKjerneelementDTO => x.asJson + case x: GrepKompetansemaalDTO => x.asJson + case x: GrepKompetansemaalSettDTO => x.asJson + case x: GrepLaererplanDTO => x.asJson + case x: GrepTverrfagligTemaDTO => x.asJson + } + // NOTE: Adding the discriminator field that scala-tsi generates in the typescript type. + // Useful for guarding the type of the object in the frontend. + json.mapObject(_.add("type", Json.fromString(result.getClass.getSimpleName))) + } + + implicit val decoder: Decoder[GrepResultDTO] = List[Decoder[GrepResultDTO]]( + Decoder[GrepKjerneelementDTO].widen, + Decoder[GrepKompetansemaalDTO].widen, + Decoder[GrepKompetansemaalSettDTO].widen, + Decoder[GrepLaererplanDTO].widen, + Decoder[GrepTverrfagligTemaDTO].widen + ).reduceLeft(_ or _) + + implicit val s: Schema[GrepResultDTO] = Schema.oneOfWrapped[GrepResultDTO] + def fromSearchable(searchable: SearchableGrepElement, language: String): Try[GrepResultDTO] = { + val titleLv = findByLanguageOrBestEffort(searchable.title.languageValues, language) + .getOrElse(LanguageValue(Language.DefaultLanguage, "")) + val title = TitleDTO(title = titleLv.value, language = titleLv.language) + + searchable.domainObject match { + case core: GrepKjerneelement => + val descriptionLvs = GrepTitle.convertTitles(core.beskrivelse.tekst.toSeq) + val descriptionLv: LanguageValue[String] = + findByLanguageOrBestEffort(descriptionLvs, language) + .getOrElse(LanguageValue(Language.DefaultLanguage, "")) + val description = DescriptionDTO(description = descriptionLv.value, language = descriptionLv.language) + + Success( + GrepKjerneelementDTO( + code = core.kode, + title = title, + description = description, + laereplan = GrepLaererplanDTO( + code = core.`tilhoerer-laereplan`.kode, + title = TitleDTO(core.`tilhoerer-laereplan`.tittel, Language.DefaultLanguage) + ) + ) + ) + case goal: GrepKompetansemaal => + Success( + GrepKompetansemaalDTO( + code = goal.kode, + title = title, + laereplan = GrepLaererplanDTO( + code = goal.`tilhoerer-laereplan`.kode, + title = TitleDTO(goal.`tilhoerer-laereplan`.tittel, Language.DefaultLanguage) + ), + kompetansemaalSett = GrepReferencedKompetansemaalSettDTO( + code = goal.`tilhoerer-kompetansemaalsett`.kode, + title = goal.`tilhoerer-kompetansemaalsett`.tittel + ), + tverrfagligeTemaer = goal.`tilknyttede-tverrfaglige-temaer`.map { crossTopic => + GrepTverrfagligTemaDTO( + code = crossTopic.referanse.kode, + title = TitleDTO(crossTopic.referanse.tittel, Language.DefaultLanguage) + ) + }, + kjerneelementer = goal.`tilknyttede-kjerneelementer`.map { core => + GrepReferencedKjerneelementDTO( + code = core.referanse.kode, + title = core.referanse.tittel + ) + } + ) + ) + case goalSet: GrepKompetansemaalSett => + Success( + GrepKompetansemaalSettDTO( + code = goalSet.kode, + title = title, + kompetansemaal = goalSet.kompetansemaal.map { goal => + GrepReferencedKompetansemaalDTO( + code = goal.kode, + title = goal.tittel + ) + } + ) + ) + case curriculum: GrepLaererplan => + Success( + GrepLaererplanDTO( + code = curriculum.kode, + title = title + ) + ) + case crossTopic: GrepTverrfagligTema => + Success( + GrepTverrfagligTemaDTO( + code = crossTopic.kode, + title = title + ) + ) + } + } } + +case class GrepReferencedKjerneelementDTO(code: String, title: String) +case class GrepReferencedKompetansemaalDTO(code: String, title: String) +case class GrepKjerneelementDTO( + code: String, + title: TitleDTO, + description: DescriptionDTO, + laereplan: GrepLaererplanDTO +) extends GrepResultDTO +case class GrepKompetansemaalDTO( + code: String, + title: TitleDTO, + laereplan: GrepLaererplanDTO, + kompetansemaalSett: GrepReferencedKompetansemaalSettDTO, + tverrfagligeTemaer: List[GrepTverrfagligTemaDTO], + kjerneelementer: List[GrepReferencedKjerneelementDTO] +) extends GrepResultDTO +case class GrepReferencedKompetansemaalSettDTO( + code: String, + title: String +) +case class GrepKompetansemaalSettDTO( + code: String, + title: TitleDTO, + kompetansemaal: List[GrepReferencedKompetansemaalDTO] +) extends GrepResultDTO +case class GrepLaererplanDTO( + code: String, + title: TitleDTO +) extends GrepResultDTO +case class GrepTverrfagligTemaDTO( + code: String, + title: TitleDTO +) extends GrepResultDTO diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala index 6b8dd65c9..3bcb7e162 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala @@ -7,26 +7,50 @@ package no.ndla.searchapi.model.grep +import cats.implicits.* import io.circe.{Decoder, Encoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.syntax.EncoderOps sealed trait GrepElement { val kode: String def getTitle: Seq[GrepTitle] } +object GrepElement { + implicit val decoder: Decoder[GrepElement] = + List[Decoder[GrepElement]]( + Decoder[GrepKjerneelement].widen, + Decoder[GrepKompetansemaal].widen, + Decoder[GrepKompetansemaalSett].widen, + Decoder[GrepLaererplan].widen, + Decoder[GrepTverrfagligTema].widen + ).reduceLeft(_ or _) + + implicit val encoder: Encoder[GrepElement] = Encoder.instance { + case x: GrepKjerneelement => x.asJson + case x: GrepKompetansemaal => x.asJson + case x: GrepKompetansemaalSett => x.asJson + case x: GrepLaererplan => x.asJson + case x: GrepTverrfagligTema => x.asJson + } +} sealed trait BelongsToLaerePlan { val `tilhoerer-laereplan`: BelongsToObj } -case class TitleObj(tekst: List[GrepTitle]) -object TitleObj { - implicit val encoder: Encoder[TitleObj] = deriveEncoder - implicit val decoder: Decoder[TitleObj] = deriveDecoder +case class GrepTextObj(tekst: List[GrepTitle]) +object GrepTextObj { + implicit val encoder: Encoder[GrepTextObj] = deriveEncoder + implicit val decoder: Decoder[GrepTextObj] = deriveDecoder } -case class GrepKjerneelement(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj) - extends GrepElement +case class GrepKjerneelement( + kode: String, + tittel: GrepTextObj, + beskrivelse: GrepTextObj, + `tilhoerer-laereplan`: BelongsToObj +) extends GrepElement with BelongsToLaerePlan { override def getTitle: Seq[GrepTitle] = tittel.tekst } @@ -35,14 +59,38 @@ object GrepKjerneelement { implicit val decoder: Decoder[GrepKjerneelement] = deriveDecoder } -case class BelongsToObj(kode: String) +case class BelongsToObj( + kode: String, + tittel: String +) object BelongsToObj { implicit val encoder: Encoder[BelongsToObj] = deriveEncoder implicit val decoder: Decoder[BelongsToObj] = deriveDecoder } -case class GrepKompetansemaal(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj) - extends GrepElement +case class ReferenceObj( + kode: String, + tittel: String +) +object ReferenceObj { + implicit val encoder: Encoder[ReferenceObj] = deriveEncoder + implicit val decoder: Decoder[ReferenceObj] = deriveDecoder +} + +case class ReferenceWrapperObj(referanse: ReferenceObj) +object ReferenceWrapperObj { + implicit val encoder: Encoder[ReferenceWrapperObj] = deriveEncoder + implicit val decoder: Decoder[ReferenceWrapperObj] = deriveDecoder +} + +case class GrepKompetansemaal( + kode: String, + tittel: GrepTextObj, + `tilhoerer-laereplan`: BelongsToObj, + `tilhoerer-kompetansemaalsett`: BelongsToObj, + `tilknyttede-tverrfaglige-temaer`: List[ReferenceWrapperObj], + `tilknyttede-kjerneelementer`: List[ReferenceWrapperObj] +) extends GrepElement with BelongsToLaerePlan { override def getTitle: Seq[GrepTitle] = tittel.tekst } @@ -51,8 +99,12 @@ object GrepKompetansemaal { implicit val decoder: Decoder[GrepKompetansemaal] = deriveDecoder } -case class GrepKompetansemaalSett(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj) - extends GrepElement +case class GrepKompetansemaalSett( + kode: String, + tittel: GrepTextObj, + `tilhoerer-laereplan`: BelongsToObj, + kompetansemaal: List[ReferenceObj] +) extends GrepElement with BelongsToLaerePlan { override def getTitle: Seq[GrepTitle] = tittel.tekst } @@ -61,7 +113,10 @@ object GrepKompetansemaalSett { implicit val decoder: Decoder[GrepKompetansemaalSett] = deriveDecoder } -case class GrepLaererplan(kode: String, tittel: TitleObj) extends GrepElement { +case class GrepLaererplan( + kode: String, + tittel: GrepTextObj +) extends GrepElement { override def getTitle: Seq[GrepTitle] = tittel.tekst } object GrepLaererplan { @@ -69,7 +124,10 @@ object GrepLaererplan { implicit val decoder: Decoder[GrepLaererplan] = deriveDecoder } -case class GrepTverrfagligTema(kode: String, tittel: Seq[GrepTitle]) extends GrepElement { +case class GrepTverrfagligTema( + kode: String, + tittel: Seq[GrepTitle] +) extends GrepElement { override def getTitle: Seq[GrepTitle] = tittel } object GrepTverrfagligTema { diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepTitle.scala b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepTitle.scala index 5448d3de9..39d763d49 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepTitle.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepTitle.scala @@ -7,12 +7,28 @@ package no.ndla.searchapi.model.grep +import com.typesafe.scalalogging.StrictLogging import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import io.circe.{Decoder, Encoder} +import no.ndla.mapping.ISO639 +import no.ndla.search.model.LanguageValue case class GrepTitle(spraak: String, verdi: String) -object GrepTitle { +object GrepTitle extends StrictLogging { implicit val encoder: Encoder[GrepTitle] = deriveEncoder implicit val decoder: Decoder[GrepTitle] = deriveDecoder + + def convertTitles(titles: Seq[GrepTitle]): Seq[LanguageValue[String]] = { + titles.flatMap(gt => { + ISO639.get6391CodeFor6392Code(gt.spraak) match { + case Some(convertedLanguage) => + Some(LanguageValue(language = convertedLanguage, value = gt.verdi.trim)) + case None if gt.spraak == "default" => None + case None => + logger.warn(s"Could not convert language code '${gt.spraak}'") + None + } + }) + } } diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/search/SearchableGrepElement.scala b/search-api/src/main/scala/no/ndla/searchapi/model/search/SearchableGrepElement.scala index 7f03e9cdf..c2727aacf 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/search/SearchableGrepElement.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/search/SearchableGrepElement.scala @@ -11,12 +11,14 @@ package no.ndla.searchapi.model.search import io.circe.{Decoder, Encoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import no.ndla.search.model.SearchableLanguageValues +import no.ndla.searchapi.model.grep.GrepElement case class SearchableGrepElement( code: String, title: SearchableLanguageValues, defaultTitle: Option[String], - laereplanCode: Option[String] + laereplanCode: Option[String], + domainObject: GrepElement ) object SearchableGrepElement { diff --git a/search-api/src/main/scala/no/ndla/searchapi/service/search/GrepIndexService.scala b/search-api/src/main/scala/no/ndla/searchapi/service/search/GrepIndexService.scala index 8d5a5f8b1..8bf57860a 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/service/search/GrepIndexService.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/service/search/GrepIndexService.scala @@ -11,6 +11,7 @@ package no.ndla.searchapi.service.search import cats.implicits.toTraverseOps import no.ndla.common.implicits.TryQuestionMark import com.sksamuel.elastic4s.ElasticDsl.* +import com.sksamuel.elastic4s.fields.ObjectField import com.sksamuel.elastic4s.requests.indexes.IndexRequest import com.sksamuel.elastic4s.requests.mappings.MappingDefinition import com.typesafe.scalalogging.StrictLogging @@ -37,7 +38,8 @@ trait GrepIndexService { val fields = List( keywordField("defaultTitle"), keywordField("code").normalizer("lower"), - keywordField("laereplanCode").normalizer("lower") + keywordField("laereplanCode").normalizer("lower"), + ObjectField("domainObject", enabled = Some(false)) ) val dynamics = generateLanguageSupportedDynamicTemplates("title", keepRaw = true) diff --git a/search-api/src/main/scala/no/ndla/searchapi/service/search/GrepSearchService.scala b/search-api/src/main/scala/no/ndla/searchapi/service/search/GrepSearchService.scala index ebd4dc103..4562f5dfa 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/service/search/GrepSearchService.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/service/search/GrepSearchService.scala @@ -17,19 +17,16 @@ import com.sksamuel.elastic4s.requests.searches.sort.SortOrder.{Asc, Desc} import com.sksamuel.elastic4s.requests.searches.{SearchHit, SearchResponse} import no.ndla.common.CirceUtil import no.ndla.common.implicits.TryQuestionMark -import no.ndla.language.Language -import no.ndla.language.Language.{AllLanguages, findByLanguageOrBestEffort} +import no.ndla.language.Language.AllLanguages import no.ndla.language.model.Iso639 -import no.ndla.search.model.LanguageValue import no.ndla.search.{BaseIndexService, Elastic4sClient} import no.ndla.searchapi.Props import no.ndla.searchapi.controller.parameters.GrepSearchInputDTO -import no.ndla.searchapi.model.api.TitleDTO import no.ndla.searchapi.model.api.grep.GrepSortDTO.* import no.ndla.searchapi.model.api.grep.{GrepResultDTO, GrepSearchResultsDTO, GrepSortDTO} import no.ndla.searchapi.model.search.{SearchType, SearchableGrepElement} -import scala.util.{Success, Try} +import scala.util.Try trait GrepSearchService { this: Props & SearchService & GrepIndexService & BaseIndexService & Elastic4sClient & SearchConverterService => @@ -153,17 +150,7 @@ trait GrepSearchService { private def hitToResult(hit: SearchHit, language: String): Try[GrepResultDTO] = { val jsonString = hit.sourceAsString val searchable = CirceUtil.tryParseAs[SearchableGrepElement](jsonString).? - val titleLv = findByLanguageOrBestEffort(searchable.title.languageValues, language) - .getOrElse(LanguageValue(Language.DefaultLanguage, "")) - val title = TitleDTO(title = titleLv.value, language = titleLv.language) - - Success( - GrepResultDTO( - code = searchable.code, - title = title, - laereplanCode = searchable.laereplanCode - ) - ) + GrepResultDTO.fromSearchable(searchable, language) } private def getGrepHits(response: RequestSuccess[SearchResponse], language: String): Try[List[GrepResultDTO]] = { diff --git a/search-api/src/main/scala/no/ndla/searchapi/service/search/SearchConverterService.scala b/search-api/src/main/scala/no/ndla/searchapi/service/search/SearchConverterService.scala index 8274eee21..5f21da900 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/service/search/SearchConverterService.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/service/search/SearchConverterService.scala @@ -261,32 +261,15 @@ trait SearchConverterService { case _ => None } val defaultTitle = grepElement.getTitle.find(_.spraak == "default") - val titles = grepElement.getTitle.flatMap(gt => { - ISO639.get6391CodeFor6392Code(gt.spraak) match { - case Some(convertedLanguage) => - Some(LanguageValue(language = convertedLanguage, value = gt.verdi.trim)) - case None if gt.spraak == "default" => None - case None => - logger.warn(s"Could not convert language code '${gt.spraak}' for grep code '${grepElement.kode}'") - None - } - }) - - def asSearchableGrep(grepElement: GrepElement): Try[SearchableGrepElement] = { - val laererplan = grepElement match { - case lp: BelongsToLaerePlan => Some(lp.tilhoerer_laereplan.kode) - case _ => None - } - val defaultTitle = grepElement.tittel.find(_.spraak == "default") - val titles = convertGrepTitleToLanguageValue(grepElement) + val titles = GrepTitle.convertTitles(grepElement.getTitle) val title = SearchableLanguageValues.fromFields(titles) - Success( SearchableGrepElement( code = grepElement.kode, title = title, defaultTitle = defaultTitle.map(_.verdi), - laereplanCode = laererplan + laereplanCode = laererplan, + domainObject = grepElement ) ) } diff --git a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala index 6a2cbda2f..ed586a706 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala @@ -58,7 +58,7 @@ import no.ndla.searchapi.model.grep.{ GrepLaererplan, GrepTitle, GrepTverrfagligTema, - TitleObj + GrepTextObj } import no.ndla.searchapi.model.search.* import no.ndla.searchapi.model.search.settings.{MultiDraftSearchSettings, SearchSettings} @@ -1587,27 +1587,27 @@ object TestData { ) val grepBundle: GrepBundle = emptyGrepBundle.copy( - kjerneelementer = List( - GrepKjerneelement( - "KE12", - TitleObj(List(GrepTitle("default", "Utforsking og problemløysing"))), - BelongsToObj("LP1") - ), - GrepKjerneelement( - "KE34", - TitleObj(List(GrepTitle("default", "Abstraksjon og generalisering"))), - BelongsToObj("LP1") - ) - ), - kompetansemaal = List( - GrepKompetansemaal( - "KM123", - TitleObj(List(GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"))), - BelongsToObj("LP1") - ) - ), - tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "Demokrati og medborgerskap")))), - laereplaner = List(GrepLaererplan("LP1", TitleObj(List(GrepTitle("default", "Læreplan i norsk (NOR01-04)"))))) +// kjerneelementer = List( +// GrepKjerneelement( +// "KE12", +// GrepTextObj(List(GrepTitle("default", "Utforsking og problemløysing"))), +// BelongsToObj("LP1") +// ), +// GrepKjerneelement( +// "KE34", +// GrepTextObj(List(GrepTitle("default", "Abstraksjon og generalisering"))), +// BelongsToObj("LP1") +// ) +// ), +// kompetansemaal = List( +// GrepKompetansemaal( +// "KM123", +// GrepTextObj(List(GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"))), +// BelongsToObj("LP1") +// ) +// ), +// tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "Demokrati og medborgerskap")))), +// laereplaner = List(GrepLaererplan("LP1", GrepTextObj(List(GrepTitle("default", "Læreplan i norsk (NOR01-04)"))))) ) val searchSettings: SearchSettings = SearchSettings( diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala index ad6e6d072..70386fd51 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala @@ -21,7 +21,7 @@ import no.ndla.searchapi.model.grep.{ GrepLaererplan, GrepTitle, GrepTverrfagligTema, - TitleObj + GrepTextObj } class GrepSearchServiceTest extends IntegrationSuite(EnableElasticsearchContainer = true) with TestEnvironment { @@ -48,49 +48,49 @@ class GrepSearchServiceTest extends IntegrationSuite(EnableElasticsearchContaine val grepTestBundle: GrepBundle = GrepBundle( kjerneelementer = List( - GrepKjerneelement( - "KE12", - TitleObj( - List(GrepTitle("default", "Utforsking og problemløysing"), GrepTitle("nob", "Utforsking og problemløsning")) - ), - BelongsToObj("LP1") - ), - GrepKjerneelement( - "KE34", - TitleObj( - List(GrepTitle("default", "Abstraksjon og generalisering"), GrepTitle("nob", "Abstraksjon og generalisering")) - ), - BelongsToObj("LP2") - ) +// GrepKjerneelement( +// "KE12", +// GrepTextObj( +// List(GrepTitle("default", "Utforsking og problemløysing"), GrepTitle("nob", "Utforsking og problemløsning")) +// ), +// BelongsToObj("LP1") +// ), +// GrepKjerneelement( +// "KE34", +// GrepTextObj( +// List(GrepTitle("default", "Abstraksjon og generalisering"), GrepTitle("nob", "Abstraksjon og generalisering")) +// ), +// BelongsToObj("LP2") +// ) ), kompetansemaal = List( - GrepKompetansemaal( - "KM123", - TitleObj( - List( - GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"), - GrepTitle("nob", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte") - ) - ), - BelongsToObj("LP2") - ) +// GrepKompetansemaal( +// "KM123", +// GrepTextObj( +// List( +// GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"), +// GrepTitle("nob", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte") +// ) +// ), +// BelongsToObj("LP2") +// ) ), kompetansemaalsett = List.empty, tverrfagligeTemaer = List( - GrepTverrfagligTema( - "TT2", - Seq(GrepTitle("default", "Demokrati og medborgerskap"), GrepTitle("nob", "Demokrati og medborgerskap")) - ) +// GrepTverrfagligTema( +// "TT2", +// Seq(GrepTitle("default", "Demokrati og medborgerskap"), GrepTitle("nob", "Demokrati og medborgerskap")) +// ) ), laereplaner = List( - GrepLaererplan( - "LP1", - TitleObj(List(GrepTitle("default", "Læreplan i norsk"), GrepTitle("nob", "Læreplan i norsk"))) - ), - GrepLaererplan( - "LP2", - TitleObj(List(GrepTitle("default", "Læreplan i engelsk"), GrepTitle("nob", "Læreplan i engelsk"))) - ) +// GrepLaererplan( +// "LP1", +// GrepTextObj(List(GrepTitle("default", "Læreplan i norsk"), GrepTitle("nob", "Læreplan i norsk"))) +// ), +// GrepLaererplan( +// "LP2", +// GrepTextObj(List(GrepTitle("default", "Læreplan i engelsk"), GrepTitle("nob", "Læreplan i engelsk"))) +// ) ) ) diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala index 6b3d8446b..af8657a81 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala @@ -19,7 +19,7 @@ import no.ndla.searchapi.model.grep.{ GrepKompetansemaal, GrepTitle, GrepTverrfagligTema, - TitleObj + GrepTextObj } import no.ndla.searchapi.model.search.{SearchTrait, SearchableArticle, SearchableGrepContext} import no.ndla.searchapi.model.taxonomy.* @@ -496,13 +496,13 @@ class SearchConverterServiceTest extends UnitSuite with TestEnvironment { test("That asSearchableDraft converts grepContexts correctly based on grepBundle if draft has grepCodes") { val draft = TestData.emptyDomainDraft.copy(id = Some(99), grepCodes = Seq("KE12", "KM123", "TT2")) val grepBundle = TestData.emptyGrepBundle.copy( - kjerneelementer = List( - GrepKjerneelement("KE12", TitleObj(List(GrepTitle("default", "tittel12"))), BelongsToObj("LP123")), - GrepKjerneelement("KE34", TitleObj(List(GrepTitle("default", "tittel34"))), BelongsToObj("LP123")) - ), - kompetansemaal = - List(GrepKompetansemaal("KM123", TitleObj(List(GrepTitle("default", "tittel123"))), BelongsToObj("LP123"))), - tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2")))) +// kjerneelementer = List( +// GrepKjerneelement("KE12", GrepTextObj(List(GrepTitle("default", "tittel12"))), BelongsToObj("LP123")), +// GrepKjerneelement("KE34", GrepTextObj(List(GrepTitle("default", "tittel34"))), BelongsToObj("LP123")) +// ), +// kompetansemaal = +// List(GrepKompetansemaal("KM123", GrepTextObj(List(GrepTitle("default", "tittel123"))), BelongsToObj("LP123"))), +// tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2")))) ) val grepContexts = List( SearchableGrepContext("KE12", Some("tittel12")), @@ -517,13 +517,13 @@ class SearchConverterServiceTest extends UnitSuite with TestEnvironment { test("That asSearchableDraft converts grepContexts correctly based on grepBundle if draft has no grepCodes") { val draft = TestData.emptyDomainDraft.copy(id = Some(99), grepCodes = Seq.empty) val grepBundle = TestData.emptyGrepBundle.copy( - kjerneelementer = List( - GrepKjerneelement("KE12", TitleObj(List(GrepTitle("default", "tittel12"))), BelongsToObj("LP123")), - GrepKjerneelement("KE34", TitleObj(List(GrepTitle("default", "tittel34"))), BelongsToObj("LP123")) - ), - kompetansemaal = - List(GrepKompetansemaal("KM123", TitleObj(List(GrepTitle("default", "tittel123"))), BelongsToObj("LP123"))), - tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", List(GrepTitle("default", "tittel2")))) +// kjerneelementer = List( +// GrepKjerneelement("KE12", GrepTextObj(List(GrepTitle("default", "tittel12"))), BelongsToObj("LP123")), +// GrepKjerneelement("KE34", GrepTextObj(List(GrepTitle("default", "tittel34"))), BelongsToObj("LP123")) +// ), +// kompetansemaal = +// List(GrepKompetansemaal("KM123", GrepTextObj(List(GrepTitle("default", "tittel123"))), BelongsToObj("LP123"))), +// tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", List(GrepTitle("default", "tittel2")))) ) val grepContexts = List.empty diff --git a/typescript/types-backend/search-api.ts b/typescript/types-backend/search-api.ts index b6f089ca7..783450b0d 100644 --- a/typescript/types-backend/search-api.ts +++ b/typescript/types-backend/search-api.ts @@ -1,5 +1,7 @@ // DO NOT EDIT: generated file by scala-tsi +export type GrepResultDTO = (IGrepKompetansemaalDTO | IGrepKjerneelementDTO | IGrepLaererplanDTO | IGrepTverrfagligTemaDTO | IGrepKompetansemaalSettDTO) + export type GrepSort = ("-relevance" | "relevance" | "-title" | "title" | "-code" | "code") export interface IApiTaxonomyContextDTO { @@ -67,6 +69,11 @@ export interface ICommentDTO { solved: boolean } +export interface IDescriptionDTO { + description: string + language: string +} + export interface IDraftResponsibleDTO { responsibleId: string lastUpdated: string @@ -110,10 +117,50 @@ export interface IDraftSearchParamsDTO { resultTypes?: SearchType[] } -export interface IGrepResultDTO { +export interface IGrepKjerneelementDTO { + code: string + title: ITitleDTO + description: IDescriptionDTO + laereplan: IGrepLaererplanDTO + type: "GrepKjerneelementDTO" +} + +export interface IGrepKompetansemaalDTO { + code: string + title: ITitleDTO + laereplan: IGrepLaererplanDTO + kompetansemaalSett: IGrepReferencedKompetansemaalSettDTO + tverrfagligeTemaer: IGrepTverrfagligTemaDTO[] + kjerneelementer: IGrepReferencedKjerneelementDTO[] + type: "GrepKompetansemaalDTO" +} + +export interface IGrepKompetansemaalSettDTO { + code: string + title: ITitleDTO + kompetansemaal: IGrepReferencedKompetansemaalDTO[] + type: "GrepKompetansemaalSettDTO" +} + +export interface IGrepLaererplanDTO { code: string title: ITitleDTO - laereplanCode?: string + type: "GrepLaererplanDTO" +} + +export interface IGrepReferencedKjerneelementDTO { + code: string + title: string +} + +export interface IGrepReferencedKompetansemaalDTO { + code: string + title: string +} + +export interface IGrepReferencedKompetansemaalSettDTO { + code: string + title: string } export interface IGrepSearchInputDTO { @@ -131,7 +178,12 @@ export interface IGrepSearchResultsDTO { page: number pageSize: number language: string - results: IGrepResultDTO[] + results: GrepResultDTO[] +} + +export interface IGrepTverrfagligTemaDTO { + code: string + title: ITitleDTO } export interface IGroupSearchResultDTO { From 939a07013851035b81647dc6ef898b566183e3cb Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Tue, 7 Jan 2025 18:19:13 +0100 Subject: [PATCH 4/8] search-api: Add `typename` discriminator hack Since scala-tsi messes up adding the `typename` discriminator we need to hack around it by adding the fields manually to the variants that does not get it :shrug: --- project/Module.scala | 10 ++++++---- project/searchapi.scala | 3 ++- .../model/api/grep/GrepResultDTO.scala | 18 +++++++++++++++--- typescript/types-backend/search-api.ts | 11 ++++++----- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/project/Module.scala b/project/Module.scala index ed4fd485d..bf4f96cc5 100644 --- a/project/Module.scala +++ b/project/Module.scala @@ -3,7 +3,8 @@ import GithubWorkflowPlugin.autoImport.* import com.scalatsi.plugin.ScalaTsiPlugin.autoImport.{ typescriptExports, typescriptGenerationImports, - typescriptOutputFile + typescriptOutputFile, + typescriptTaggedUnionDiscriminator } import org.scalafmt.sbt.ScalafmtPlugin.autoImport.* import org.typelevel.sbt.tpolecat.TpolecatPlugin.autoImport.* @@ -192,9 +193,10 @@ trait Module { protected def typescriptSettings(imports: Seq[String], exports: Seq[String]) = { Seq( - typescriptGenerationImports := imports, - typescriptExports := exports, - typescriptOutputFile := file("./typescript/types-backend") / s"${this.moduleName}.ts" + typescriptGenerationImports := imports, + typescriptExports := exports, + typescriptOutputFile := file("./typescript/types-backend") / s"${this.moduleName}.ts", + typescriptTaggedUnionDiscriminator := Some("typename") ) } } diff --git a/project/searchapi.scala b/project/searchapi.scala index 5191b2582..38f458e86 100644 --- a/project/searchapi.scala +++ b/project/searchapi.scala @@ -43,7 +43,8 @@ object searchapi extends Module { "SubjectAggregationsDTO", "SubjectAggsInputDTO", "GrepSearchInputDTO", - "grep.GrepSearchResultsDTO" + "grep.GrepSearchResultsDTO", + "grep.GrepResultDTO" ) ) diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala index c5eb38010..a0cc254d1 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala @@ -9,8 +9,14 @@ package no.ndla.searchapi.model.api.grep import cats.implicits.* +import com.scalatsi.TSType.fromCaseClass +import com.scalatsi.TypescriptType.{TSLiteralString, TSString, TSUnion} +import com.scalatsi.{TSIType, TSNamedType, TSType} +import com.scalatsi.dsl.* import io.circe.generic.auto.* import sttp.tapir.generic.auto.* + +import scala.reflect.runtime.universe.* import io.circe.syntax.* import io.circe.{Decoder, Encoder, Json} import no.ndla.language.Language @@ -29,6 +35,7 @@ import no.ndla.searchapi.model.search.SearchableGrepElement import sttp.tapir.Schema import sttp.tapir.Schema.annotations.description +import scala.reflect.ClassTag import scala.util.{Success, Try} @description("Information about a single grep search result entry") @@ -48,9 +55,12 @@ object GrepResultDTO { } // NOTE: Adding the discriminator field that scala-tsi generates in the typescript type. // Useful for guarding the type of the object in the frontend. - json.mapObject(_.add("type", Json.fromString(result.getClass.getSimpleName))) + json.mapObject(_.add("typename", Json.fromString(result.getClass.getSimpleName))) } + implicit val s1: Schema["GrepLaererplanDTO"] = Schema.string + implicit val s2: Schema["GrepTverrfagligTemaDTO"] = Schema.string + implicit val decoder: Decoder[GrepResultDTO] = List[Decoder[GrepResultDTO]]( Decoder[GrepKjerneelementDTO].widen, Decoder[GrepKompetansemaalDTO].widen, @@ -169,9 +179,11 @@ case class GrepKompetansemaalSettDTO( ) extends GrepResultDTO case class GrepLaererplanDTO( code: String, - title: TitleDTO + title: TitleDTO, + typename: "GrepLaererplanDTO" = "GrepLaererplanDTO" ) extends GrepResultDTO case class GrepTverrfagligTemaDTO( code: String, - title: TitleDTO + title: TitleDTO, + typename: "GrepTverrfagligTemaDTO" = "GrepTverrfagligTemaDTO" ) extends GrepResultDTO diff --git a/typescript/types-backend/search-api.ts b/typescript/types-backend/search-api.ts index 783450b0d..9de900a90 100644 --- a/typescript/types-backend/search-api.ts +++ b/typescript/types-backend/search-api.ts @@ -1,6 +1,6 @@ // DO NOT EDIT: generated file by scala-tsi -export type GrepResultDTO = (IGrepKompetansemaalDTO | IGrepKjerneelementDTO | IGrepLaererplanDTO | IGrepTverrfagligTemaDTO | IGrepKompetansemaalSettDTO) +export type GrepResultDTO = (IGrepKjerneelementDTO | IGrepKompetansemaalSettDTO | IGrepTverrfagligTemaDTO | IGrepKompetansemaalDTO | IGrepLaererplanDTO) export type GrepSort = ("-relevance" | "relevance" | "-title" | "title" | "-code" | "code") @@ -122,7 +122,7 @@ export interface IGrepKjerneelementDTO { title: ITitleDTO description: IDescriptionDTO laereplan: IGrepLaererplanDTO - type: "GrepKjerneelementDTO" + typename: "GrepKjerneelementDTO" } export interface IGrepKompetansemaalDTO { @@ -132,20 +132,20 @@ export interface IGrepKompetansemaalDTO { kompetansemaalSett: IGrepReferencedKompetansemaalSettDTO tverrfagligeTemaer: IGrepTverrfagligTemaDTO[] kjerneelementer: IGrepReferencedKjerneelementDTO[] - type: "GrepKompetansemaalDTO" + typename: "GrepKompetansemaalDTO" } export interface IGrepKompetansemaalSettDTO { code: string title: ITitleDTO kompetansemaal: IGrepReferencedKompetansemaalDTO[] - type: "GrepKompetansemaalSettDTO" + typename: "GrepKompetansemaalSettDTO" } export interface IGrepLaererplanDTO { code: string title: ITitleDTO - type: "GrepLaererplanDTO" + typename: "GrepLaererplanDTO" } export interface IGrepReferencedKjerneelementDTO { @@ -184,6 +184,7 @@ export interface IGrepSearchResultsDTO { export interface IGrepTverrfagligTemaDTO { code: string title: ITitleDTO + typename: "GrepTverrfagligTemaDTO" } export interface IGroupSearchResultDTO { From ffad59a5fa122606af88afbbd5a98aacc8ad3ef0 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Wed, 8 Jan 2025 06:44:28 +0100 Subject: [PATCH 5/8] search-api: Fix grep tests after rewriting to dump fetching I broke some tests in the rewrite to fetching grep information via the zipped dump. This patch fixes those. --- .../model/api/grep/GrepResultDTO.scala | 6 -- .../scala/no/ndla/searchapi/TestData.scala | 49 +++++++----- .../integration/GrepApiClientTest.scala | 21 ----- .../search/GrepSearchServiceTest.scala | 77 ++++++++++--------- .../search/SearchConverterServiceTest.scala | 64 +++++++++++---- 5 files changed, 119 insertions(+), 98 deletions(-) delete mode 100644 search-api/src/test/scala/no/ndla/searchapi/integration/GrepApiClientTest.scala diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala index a0cc254d1..17d47f629 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala @@ -9,14 +9,9 @@ package no.ndla.searchapi.model.api.grep import cats.implicits.* -import com.scalatsi.TSType.fromCaseClass -import com.scalatsi.TypescriptType.{TSLiteralString, TSString, TSUnion} -import com.scalatsi.{TSIType, TSNamedType, TSType} -import com.scalatsi.dsl.* import io.circe.generic.auto.* import sttp.tapir.generic.auto.* -import scala.reflect.runtime.universe.* import io.circe.syntax.* import io.circe.{Decoder, Encoder, Json} import no.ndla.language.Language @@ -35,7 +30,6 @@ import no.ndla.searchapi.model.search.SearchableGrepElement import sttp.tapir.Schema import sttp.tapir.Schema.annotations.description -import scala.reflect.ClassTag import scala.util.{Success, Try} @description("Information about a single grep search result entry") diff --git a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala index ed586a706..a8732bc33 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala @@ -1587,27 +1587,34 @@ object TestData { ) val grepBundle: GrepBundle = emptyGrepBundle.copy( -// kjerneelementer = List( -// GrepKjerneelement( -// "KE12", -// GrepTextObj(List(GrepTitle("default", "Utforsking og problemløysing"))), -// BelongsToObj("LP1") -// ), -// GrepKjerneelement( -// "KE34", -// GrepTextObj(List(GrepTitle("default", "Abstraksjon og generalisering"))), -// BelongsToObj("LP1") -// ) -// ), -// kompetansemaal = List( -// GrepKompetansemaal( -// "KM123", -// GrepTextObj(List(GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"))), -// BelongsToObj("LP1") -// ) -// ), -// tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "Demokrati og medborgerskap")))), -// laereplaner = List(GrepLaererplan("LP1", GrepTextObj(List(GrepTitle("default", "Læreplan i norsk (NOR01-04)"))))) + kjerneelementer = List( + GrepKjerneelement( + kode = "KE12", + tittel = GrepTextObj(List(GrepTitle("default", "Utforsking og problemløysing"))), + beskrivelse = GrepTextObj(List(GrepTitle("default", ""))), + `tilhoerer-laereplan` = BelongsToObj("LP1", "Dette er LP1") + ), + GrepKjerneelement( + kode = "KE34", + tittel = GrepTextObj(List(GrepTitle("default", "Abstraksjon og generalisering"))), + beskrivelse = GrepTextObj(List(GrepTitle("default", ""))), + `tilhoerer-laereplan` = BelongsToObj("LP1", "Dette er LP2") + ) + ), + kompetansemaal = List( + GrepKompetansemaal( + kode = "KM123", + tittel = GrepTextObj( + List(GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte")) + ), + `tilhoerer-laereplan` = BelongsToObj("LP1", "Dette er LP1"), + `tilhoerer-kompetansemaalsett` = BelongsToObj("KMS1", "Dette er KMS1"), + `tilknyttede-tverrfaglige-temaer` = List(), + `tilknyttede-kjerneelementer` = List() + ) + ), + tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "Demokrati og medborgerskap")))), + laereplaner = List(GrepLaererplan("LP1", GrepTextObj(List(GrepTitle("default", "Læreplan i norsk (NOR01-04)"))))) ) val searchSettings: SearchSettings = SearchSettings( diff --git a/search-api/src/test/scala/no/ndla/searchapi/integration/GrepApiClientTest.scala b/search-api/src/test/scala/no/ndla/searchapi/integration/GrepApiClientTest.scala deleted file mode 100644 index 06428bb59..000000000 --- a/search-api/src/test/scala/no/ndla/searchapi/integration/GrepApiClientTest.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Part of NDLA backend.search-api.test - * Copyright (C) 2024 NDLA - * - * See LICENSE - * - */ - -package no.ndla.searchapi.integration - -import no.ndla.searchapi.{TestEnvironment, UnitSuite} - -class GrepApiClientTest extends UnitSuite with TestEnvironment { - - test("do stuff") { - - val x = ??? - - } - -} diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala index 70386fd51..97532d6cb 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala @@ -48,49 +48,54 @@ class GrepSearchServiceTest extends IntegrationSuite(EnableElasticsearchContaine val grepTestBundle: GrepBundle = GrepBundle( kjerneelementer = List( -// GrepKjerneelement( -// "KE12", -// GrepTextObj( -// List(GrepTitle("default", "Utforsking og problemløysing"), GrepTitle("nob", "Utforsking og problemløsning")) -// ), -// BelongsToObj("LP1") -// ), -// GrepKjerneelement( -// "KE34", -// GrepTextObj( -// List(GrepTitle("default", "Abstraksjon og generalisering"), GrepTitle("nob", "Abstraksjon og generalisering")) -// ), -// BelongsToObj("LP2") -// ) + GrepKjerneelement( + "KE12", + GrepTextObj( + List(GrepTitle("default", "Utforsking og problemløysing"), GrepTitle("nob", "Utforsking og problemløsning")) + ), + GrepTextObj(List(GrepTitle("default", ""))), + BelongsToObj("LP1", "Dette er LP1") + ), + GrepKjerneelement( + "KE34", + GrepTextObj( + List(GrepTitle("default", "Abstraksjon og generalisering"), GrepTitle("nob", "Abstraksjon og generalisering")) + ), + GrepTextObj(List(GrepTitle("default", ""))), + BelongsToObj("LP2", "Dette er LP2") + ) ), kompetansemaal = List( -// GrepKompetansemaal( -// "KM123", -// GrepTextObj( -// List( -// GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"), -// GrepTitle("nob", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte") -// ) -// ), -// BelongsToObj("LP2") -// ) + GrepKompetansemaal( + kode = "KM123", + tittel = GrepTextObj( + List( + GrepTitle("default", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte"), + GrepTitle("nob", "bruke ulike kilder på en kritisk, hensiktsmessig og etterrettelig måte") + ) + ), + `tilhoerer-laereplan` = BelongsToObj("LP2", "Dette er LP2"), + `tilhoerer-kompetansemaalsett` = BelongsToObj("KE200", "Kompetansemaalsett"), + `tilknyttede-tverrfaglige-temaer` = List(), + `tilknyttede-kjerneelementer` = List() + ) ), kompetansemaalsett = List.empty, tverrfagligeTemaer = List( -// GrepTverrfagligTema( -// "TT2", -// Seq(GrepTitle("default", "Demokrati og medborgerskap"), GrepTitle("nob", "Demokrati og medborgerskap")) -// ) + GrepTverrfagligTema( + "TT2", + Seq(GrepTitle("default", "Demokrati og medborgerskap"), GrepTitle("nob", "Demokrati og medborgerskap")) + ) ), laereplaner = List( -// GrepLaererplan( -// "LP1", -// GrepTextObj(List(GrepTitle("default", "Læreplan i norsk"), GrepTitle("nob", "Læreplan i norsk"))) -// ), -// GrepLaererplan( -// "LP2", -// GrepTextObj(List(GrepTitle("default", "Læreplan i engelsk"), GrepTitle("nob", "Læreplan i engelsk"))) -// ) + GrepLaererplan( + "LP1", + GrepTextObj(List(GrepTitle("default", "Læreplan i norsk"), GrepTitle("nob", "Læreplan i norsk"))) + ), + GrepLaererplan( + "LP2", + GrepTextObj(List(GrepTitle("default", "Læreplan i engelsk"), GrepTitle("nob", "Læreplan i engelsk"))) + ) ) ) diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala index af8657a81..a3a8baa39 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala @@ -496,13 +496,31 @@ class SearchConverterServiceTest extends UnitSuite with TestEnvironment { test("That asSearchableDraft converts grepContexts correctly based on grepBundle if draft has grepCodes") { val draft = TestData.emptyDomainDraft.copy(id = Some(99), grepCodes = Seq("KE12", "KM123", "TT2")) val grepBundle = TestData.emptyGrepBundle.copy( -// kjerneelementer = List( -// GrepKjerneelement("KE12", GrepTextObj(List(GrepTitle("default", "tittel12"))), BelongsToObj("LP123")), -// GrepKjerneelement("KE34", GrepTextObj(List(GrepTitle("default", "tittel34"))), BelongsToObj("LP123")) -// ), -// kompetansemaal = -// List(GrepKompetansemaal("KM123", GrepTextObj(List(GrepTitle("default", "tittel123"))), BelongsToObj("LP123"))), -// tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2")))) + kjerneelementer = List( + GrepKjerneelement( + "KE12", + GrepTextObj(List(GrepTitle("default", "tittel12"))), + GrepTextObj(List(GrepTitle("default", ""))), + BelongsToObj("LP123", "Dette er LP123") + ), + GrepKjerneelement( + "KE34", + GrepTextObj(List(GrepTitle("default", "tittel34"))), + GrepTextObj(List(GrepTitle("default", ""))), + BelongsToObj("LP123", "Dette er LP123") + ) + ), + kompetansemaal = List( + GrepKompetansemaal( + "KM123", + GrepTextObj(List(GrepTitle("default", "tittel123"))), + BelongsToObj("LP123", "Dette er LP123"), + BelongsToObj("KMS123", "Dette er KMS123"), + List(), + List() + ) + ), + tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2")))) ) val grepContexts = List( SearchableGrepContext("KE12", Some("tittel12")), @@ -517,13 +535,31 @@ class SearchConverterServiceTest extends UnitSuite with TestEnvironment { test("That asSearchableDraft converts grepContexts correctly based on grepBundle if draft has no grepCodes") { val draft = TestData.emptyDomainDraft.copy(id = Some(99), grepCodes = Seq.empty) val grepBundle = TestData.emptyGrepBundle.copy( -// kjerneelementer = List( -// GrepKjerneelement("KE12", GrepTextObj(List(GrepTitle("default", "tittel12"))), BelongsToObj("LP123")), -// GrepKjerneelement("KE34", GrepTextObj(List(GrepTitle("default", "tittel34"))), BelongsToObj("LP123")) -// ), -// kompetansemaal = -// List(GrepKompetansemaal("KM123", GrepTextObj(List(GrepTitle("default", "tittel123"))), BelongsToObj("LP123"))), -// tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", List(GrepTitle("default", "tittel2")))) + kjerneelementer = List( + GrepKjerneelement( + "KE12", + GrepTextObj(List(GrepTitle("default", "tittel12"))), + GrepTextObj(List(GrepTitle("default", ""))), + BelongsToObj("LP123", "Dette er LP123") + ), + GrepKjerneelement( + "KE34", + GrepTextObj(List(GrepTitle("default", "tittel34"))), + GrepTextObj(List(GrepTitle("default", ""))), + BelongsToObj("LP123", "Dette er LP123") + ) + ), + kompetansemaal = List( + GrepKompetansemaal( + "KM123", + GrepTextObj(List(GrepTitle("default", "tittel123"))), + BelongsToObj("LP123", "Dette er LP123"), + BelongsToObj("KMS123", "Dette er KMS123"), + List(), + List() + ) + ), + tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2")))) ) val grepContexts = List.empty From 2b1be315e9cede08cb581a2787ed6d5be2639eeb Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Fri, 10 Jan 2025 09:13:54 +0100 Subject: [PATCH 6/8] Fix copyright year --- .../src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala | 2 +- .../main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala b/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala index 1245bf8fa..fe39cf165 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala @@ -1,6 +1,6 @@ /* * Part of NDLA search-api - * Copyright (C) 2024 NDLA + * Copyright (C) 2025 NDLA * * See LICENSE * diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala b/search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala index 31d044fa3..d47462e18 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala @@ -1,6 +1,6 @@ /* * Part of NDLA search-api - * Copyright (C) 2018 NDLA + * Copyright (C) 2025 NDLA * * See LICENSE */ From 4d7b30da5da44f9adce97b34cfbf6f76d94f28a3 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Mon, 13 Jan 2025 08:47:53 +0100 Subject: [PATCH 7/8] search-api: Add `replacedBy` field to curriculums --- .../model/api/grep/GrepResultDTO.scala | 22 +++++++++++++------ .../searchapi/model/grep/GrepElement.scala | 3 ++- .../scala/no/ndla/searchapi/TestData.scala | 8 ++++++- .../search/GrepSearchServiceTest.scala | 6 +++-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala index 17d47f629..2699a5f6b 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala @@ -82,9 +82,9 @@ object GrepResultDTO { code = core.kode, title = title, description = description, - laereplan = GrepLaererplanDTO( + laereplan = GrepReferencedLaereplanDTO( code = core.`tilhoerer-laereplan`.kode, - title = TitleDTO(core.`tilhoerer-laereplan`.tittel, Language.DefaultLanguage) + title = core.`tilhoerer-laereplan`.tittel ) ) ) @@ -93,9 +93,9 @@ object GrepResultDTO { GrepKompetansemaalDTO( code = goal.kode, title = title, - laereplan = GrepLaererplanDTO( + laereplan = GrepReferencedLaereplanDTO( code = goal.`tilhoerer-laereplan`.kode, - title = TitleDTO(goal.`tilhoerer-laereplan`.tittel, Language.DefaultLanguage) + title = goal.`tilhoerer-laereplan`.tittel ), kompetansemaalSett = GrepReferencedKompetansemaalSettDTO( code = goal.`tilhoerer-kompetansemaalsett`.kode, @@ -132,7 +132,13 @@ object GrepResultDTO { Success( GrepLaererplanDTO( code = curriculum.kode, - title = title + title = title, + replacedBy = curriculum.`erstattes-av`.map(replacement => + GrepReferencedLaereplanDTO( + code = replacement.kode, + title = replacement.tittel + ) + ) ) ) case crossTopic: GrepTverrfagligTema => @@ -148,16 +154,17 @@ object GrepResultDTO { case class GrepReferencedKjerneelementDTO(code: String, title: String) case class GrepReferencedKompetansemaalDTO(code: String, title: String) +case class GrepReferencedLaereplanDTO(code: String, title: String) case class GrepKjerneelementDTO( code: String, title: TitleDTO, description: DescriptionDTO, - laereplan: GrepLaererplanDTO + laereplan: GrepReferencedLaereplanDTO ) extends GrepResultDTO case class GrepKompetansemaalDTO( code: String, title: TitleDTO, - laereplan: GrepLaererplanDTO, + laereplan: GrepReferencedLaereplanDTO, kompetansemaalSett: GrepReferencedKompetansemaalSettDTO, tverrfagligeTemaer: List[GrepTverrfagligTemaDTO], kjerneelementer: List[GrepReferencedKjerneelementDTO] @@ -174,6 +181,7 @@ case class GrepKompetansemaalSettDTO( case class GrepLaererplanDTO( code: String, title: TitleDTO, + replacedBy: List[GrepReferencedLaereplanDTO], typename: "GrepLaererplanDTO" = "GrepLaererplanDTO" ) extends GrepResultDTO case class GrepTverrfagligTemaDTO( diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala index 3bcb7e162..3be6f12a1 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala @@ -115,7 +115,8 @@ object GrepKompetansemaalSett { case class GrepLaererplan( kode: String, - tittel: GrepTextObj + tittel: GrepTextObj, + `erstattes-av`: List[ReferenceObj] ) extends GrepElement { override def getTitle: Seq[GrepTitle] = tittel.tekst } diff --git a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala index a8732bc33..b45b24d44 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala @@ -1614,7 +1614,13 @@ object TestData { ) ), tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "Demokrati og medborgerskap")))), - laereplaner = List(GrepLaererplan("LP1", GrepTextObj(List(GrepTitle("default", "Læreplan i norsk (NOR01-04)"))))) + laereplaner = List( + GrepLaererplan( + "LP1", + GrepTextObj(List(GrepTitle("default", "Læreplan i norsk (NOR01-04)"))), + `erstattes-av` = List.empty + ) + ) ) val searchSettings: SearchSettings = SearchSettings( diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala index 97532d6cb..7f9179104 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala @@ -90,11 +90,13 @@ class GrepSearchServiceTest extends IntegrationSuite(EnableElasticsearchContaine laereplaner = List( GrepLaererplan( "LP1", - GrepTextObj(List(GrepTitle("default", "Læreplan i norsk"), GrepTitle("nob", "Læreplan i norsk"))) + GrepTextObj(List(GrepTitle("default", "Læreplan i norsk"), GrepTitle("nob", "Læreplan i norsk"))), + List.empty ), GrepLaererplan( "LP2", - GrepTextObj(List(GrepTitle("default", "Læreplan i engelsk"), GrepTitle("nob", "Læreplan i engelsk"))) + GrepTextObj(List(GrepTitle("default", "Læreplan i engelsk"), GrepTitle("nob", "Læreplan i engelsk"))), + List.empty ) ) ) From 84598349a354ded5d55d28f786570957f66579fb Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Mon, 13 Jan 2025 09:11:17 +0100 Subject: [PATCH 8/8] search-api: Add `reuseOf` field to competencegoals --- .../no/ndla/searchapi/model/api/grep/GrepResultDTO.scala | 9 ++++++++- .../scala/no/ndla/searchapi/model/grep/GrepElement.scala | 3 ++- .../src/test/scala/no/ndla/searchapi/TestData.scala | 3 ++- .../searchapi/service/search/GrepSearchServiceTest.scala | 3 ++- .../service/search/SearchConverterServiceTest.scala | 6 ++++-- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala index 2699a5f6b..273917004 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala @@ -112,6 +112,12 @@ object GrepResultDTO { code = core.referanse.kode, title = core.referanse.tittel ) + }, + reuseOf = goal.`gjenbruk-av`.map { goal => + GrepReferencedKompetansemaalDTO( + code = goal.kode, + title = goal.tittel + ) } ) ) @@ -167,7 +173,8 @@ case class GrepKompetansemaalDTO( laereplan: GrepReferencedLaereplanDTO, kompetansemaalSett: GrepReferencedKompetansemaalSettDTO, tverrfagligeTemaer: List[GrepTverrfagligTemaDTO], - kjerneelementer: List[GrepReferencedKjerneelementDTO] + kjerneelementer: List[GrepReferencedKjerneelementDTO], + reuseOf: Option[GrepReferencedKompetansemaalDTO] ) extends GrepResultDTO case class GrepReferencedKompetansemaalSettDTO( code: String, diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala index 3be6f12a1..693061499 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala @@ -89,7 +89,8 @@ case class GrepKompetansemaal( `tilhoerer-laereplan`: BelongsToObj, `tilhoerer-kompetansemaalsett`: BelongsToObj, `tilknyttede-tverrfaglige-temaer`: List[ReferenceWrapperObj], - `tilknyttede-kjerneelementer`: List[ReferenceWrapperObj] + `tilknyttede-kjerneelementer`: List[ReferenceWrapperObj], + `gjenbruk-av`: Option[ReferenceObj] ) extends GrepElement with BelongsToLaerePlan { override def getTitle: Seq[GrepTitle] = tittel.tekst diff --git a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala index b45b24d44..44281826c 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala @@ -1610,7 +1610,8 @@ object TestData { `tilhoerer-laereplan` = BelongsToObj("LP1", "Dette er LP1"), `tilhoerer-kompetansemaalsett` = BelongsToObj("KMS1", "Dette er KMS1"), `tilknyttede-tverrfaglige-temaer` = List(), - `tilknyttede-kjerneelementer` = List() + `tilknyttede-kjerneelementer` = List(), + `gjenbruk-av` = None ) ), tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "Demokrati og medborgerskap")))), diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala index 7f9179104..cdc235feb 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/GrepSearchServiceTest.scala @@ -77,7 +77,8 @@ class GrepSearchServiceTest extends IntegrationSuite(EnableElasticsearchContaine `tilhoerer-laereplan` = BelongsToObj("LP2", "Dette er LP2"), `tilhoerer-kompetansemaalsett` = BelongsToObj("KE200", "Kompetansemaalsett"), `tilknyttede-tverrfaglige-temaer` = List(), - `tilknyttede-kjerneelementer` = List() + `tilknyttede-kjerneelementer` = List(), + `gjenbruk-av` = None ) ), kompetansemaalsett = List.empty, diff --git a/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala b/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala index a3a8baa39..388509de6 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/service/search/SearchConverterServiceTest.scala @@ -517,7 +517,8 @@ class SearchConverterServiceTest extends UnitSuite with TestEnvironment { BelongsToObj("LP123", "Dette er LP123"), BelongsToObj("KMS123", "Dette er KMS123"), List(), - List() + List(), + None ) ), tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2")))) @@ -556,7 +557,8 @@ class SearchConverterServiceTest extends UnitSuite with TestEnvironment { BelongsToObj("LP123", "Dette er LP123"), BelongsToObj("KMS123", "Dette er KMS123"), List(), - List() + List(), + None ) ), tverrfagligeTemaer = List(GrepTverrfagligTema("TT2", Seq(GrepTitle("default", "tittel2"))))