From 2604e167880f2715bf1ba289f44ee889854b0e72 Mon Sep 17 00:00:00 2001 From: Simao Mata Date: Thu, 1 Aug 2024 10:39:11 +0100 Subject: [PATCH 1/3] Support search for multiple hardwareIds --- .gitlab-ci.yml | 2 +- .../tuf/reposerver/http/PackageSearch.scala | 11 ++--------- .../http/RepoTargetsResourceSpec.scala | 17 ++++++++++++++++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8031b333..78cc9110 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ test: services: - name: uptane/tuf-nginx:latest alias: tuf-nginx - - name: mariadb:10.4 + - name: mariadb:10.11 alias: db command: - --character-set-server=utf8 diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala index 2f8b7f79..75bd1852 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala @@ -105,8 +105,7 @@ class PackageSearch()(implicit db: Database) { IF(${searchParams.name.isEmpty}, true, name = ${searchParams.name}) AND IF(${searchParams.version.isEmpty}, true, version = ${searchParams.version}) AND IF(${searchParams.nameContains.isEmpty}, true, LOCATE(${searchParams.nameContains}, name) > 0) AND - IF(${searchParams.hardwareIds.isEmpty}, true, JSON_CONTAINS(hardwareids, JSON_QUOTE(${searchParams.hardwareIds.headOption - .map(_.value)}))) AND + IF(${searchParams.hardwareIds.isEmpty}, true, JSON_OVERLAPS(${searchParams.hardwareIds}, hardwareids) = 1) AND IF(${searchParams.hashes.isEmpty}, true, FIND_IN_SET(JSON_UNQUOTE(JSON_EXTRACT(checksum, '$$.hash')), ${searchParams.hashes .map(_.value)}) > 0) ORDER BY @@ -115,11 +114,6 @@ class PackageSearch()(implicit db: Database) { length LIMIT $limit OFFSET $offset""".as[Q] - // TODO: This needs a newer mariadb version - // hardwareId filter should be: - // IF(${searchParams.hardwareIds.isEmpty}, true, JSON_LENGTH(JSON_ARRAY_INTERSECT(hardwareids, ${searchParams.hardwareIds})) > 0) - // but JSON_ARRAY_INTERSECT is not supported in our mariadb version - querySqlAction } @@ -253,8 +247,7 @@ class PackageSearch()(implicit db: Database) { IF(${searchParams.origin.isEmpty}, true, FIND_IN_SET(origin, ${searchParams.origin}) > 0) AND IF(${searchParams.nameContains.isEmpty}, true, LOCATE(${searchParams.nameContains}, name) > 0) AND IF(${searchParams.version.isEmpty}, true, version = ${searchParams.version}) AND - IF(${searchParams.hardwareIds.isEmpty}, true, JSON_CONTAINS(hardwareids, JSON_QUOTE(${searchParams.hardwareIds.headOption - .map(_.value)}))) AND + IF(${searchParams.hardwareIds.isEmpty}, true, JSON_OVERLAPS(${searchParams.hardwareIds}, hardwareids) = 1) AND IF(${searchParams.hashes.isEmpty}, true, FIND_IN_SET(JSON_UNQUOTE(JSON_EXTRACT(checksum, '$$.hash')), ${searchParams.hashes .map(_.value)}) > 0) GROUP BY name diff --git a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala index 93ee7823..67946dae 100644 --- a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala +++ b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala @@ -370,7 +370,6 @@ class RepoTargetsResourceSpec } } - // TODO: Currently only filters by single hardwareid testWithRepo("filters by hardwareIds") { implicit ns => implicit repoId => addTargetToRepo(repoId) @@ -406,6 +405,14 @@ class RepoTargetsResourceSpec val values = responseAs[PaginationResult[Package]].values values shouldBe empty } + + Get( + apiUriV2(s"user_repo/search?hardwareIds=delegated-hardware-id-001,myid001") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val values = responseAs[PaginationResult[Package]].values + values should have size 2 + } } testWithRepo("grouped-search gets packages aggregated by version") { @@ -472,6 +479,14 @@ class RepoTargetsResourceSpec values should have size 1 } + Get( + apiUriV2(s"user_repo/grouped-search?hardwareIds=delegated-hardware-id-001,myid001") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val values = responseAs[PaginationResult[AggregatedPackage]].values + values should have size 2 + } + Get( apiUriV2(s"user_repo/grouped-search?hardwareIds=somethingelse") ).namespaced ~> routes ~> check { From c5b51579c39c5036ca870833bb6cf40cffd273d0 Mon Sep 17 00:00:00 2001 From: Simao Mata Date: Tue, 27 Aug 2024 16:16:30 +0100 Subject: [PATCH 2/3] Add new endpoint to search for hardwareids for which at least one package exists ``` GET `/user_repo/hardwareids-packages` returns: { "values": ["myid01"], "total": 1, "offset: 0", "limit": 1 } ``` Note: At the moment, all results will be returned on the first page, regardless of the provided pagination parameters. --- .../tuf/reposerver/http/PackageSearch.scala | 19 ++++++ .../reposerver/http/RepoTargetsResource.scala | 23 +++++-- .../http/RepoTargetsResourceSpec.scala | 64 ++++++++++++++++--- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala index 75bd1852..75dc5c87 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala @@ -257,6 +257,25 @@ class PackageSearch()(implicit db: Database) { LIMIT $limit OFFSET $offset""".as[Q] db.run(q) + + } + + def hardwareIdsWithPackages(repoId: RepoId): Future[Seq[HardwareIdentifier]] = { + implicit val getResult: GetResult[HardwareIdentifier] = GetResult.GetString.andThen { str => + refineV[ValidHardwareIdentifier](str) + .valueOr(msg => + throw new IllegalArgumentException(s"hardwareid not properly formatted: $str: $msg") + ) + } + + val q = + sql""" + select distinct hwid from aggregated_items a, + json_table(hardwareids, '$$[*]' columns(hwid varchar(255) path '$$')) t1 + where a.repo_id = ${repoId.show} + """.as[HardwareIdentifier] + + db.run(q) } } diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala index 57231fbd..f03f77a5 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala @@ -25,6 +25,7 @@ import eu.timepit.refined.refineV import slick.jdbc.MySQLProfile.api.* import com.advancedtelematic.libats.http.RefinedMarshallingSupport.* import scala.concurrent.ExecutionContext +import com.advancedtelematic.libats.codecs.CirceRefined.* case class PackageSearchParameters(origin: Seq[String], nameContains: Option[String], @@ -79,21 +80,33 @@ class RepoTargetsResource(namespaceValidation: NamespaceValidation)( name, version, hardwareIds.getOrElse(Seq.empty), - hashes.getOrElse(Seq.empty), + hashes.getOrElse(Seq.empty) ) ) } + private var packageSearch = new PackageSearch() + // format: off + val route = (pathPrefix("user_repo") & NamespaceRepoId(namespaceValidation, repoNamespaceRepo.findFor)) { repoId => + packageSearch = new PackageSearch() concat( + path("hardwareids-packages") { + get { + val f = packageSearch.hardwareIdsWithPackages(repoId).map { values => + PaginationResult(values, values.length, 0, values.length) + } + complete(f) + } + }, path("search") { (get & PaginationParams & SearchParams & SortByTargetItemsParam) { case (offset, limit, searchParams, sortBy, sortDirection) => val f = for { - count <- (new PackageSearch()).count(repoId, searchParams) - values <- (new PackageSearch()).find(repoId, offset, limit, searchParams, sortBy, sortDirection) + count <- packageSearch.count(repoId, searchParams) + values <- packageSearch.find(repoId, offset, limit, searchParams, sortBy, sortDirection) } yield { PaginationResult(values.map(_.transformInto[ClientPackage]), count, offset, limit) } @@ -104,8 +117,8 @@ class RepoTargetsResource(namespaceValidation: NamespaceValidation)( path("grouped-search") { (get & PaginationParams & SearchParams & SortByAggregatedTargetItemsParam) { case (offset, limit, searchParams, sortBy, sortDirection) => val f = for { - count <- (new PackageSearch()).findAggregatedCount(repoId, searchParams) - values <- (new PackageSearch()).findAggregated(repoId, offset, limit, searchParams, sortBy, sortDirection) + count <- packageSearch.findAggregatedCount(repoId, searchParams) + values <- packageSearch.findAggregated(repoId, offset, limit, searchParams, sortBy, sortDirection) } yield PaginationResult(values.map(_.transformInto[ClientAggregatedPackage]), count, offset, limit) diff --git a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala index 67946dae..cd3c1a59 100644 --- a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala +++ b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala @@ -8,21 +8,18 @@ import org.scalatest.OptionValues.* import akka.http.scaladsl.model.{HttpEntity, StatusCodes} import akka.util.ByteString import com.advancedtelematic.libats.data.PaginationResult -import com.advancedtelematic.tuf.reposerver.util.{ - RepoResourceDelegationsSpecUtil, - ResourceSpec, - TufReposerverSpec -} +import com.advancedtelematic.tuf.reposerver.util.{RepoResourceDelegationsSpecUtil, ResourceSpec, TufReposerverSpec} import com.advancedtelematic.tuf.reposerver.util.NamespaceSpecOps.* import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* import com.advancedtelematic.tuf.reposerver.data.RepoDataType.* import com.advancedtelematic.tuf.reposerver.data.RepoDataType.Package.* import com.advancedtelematic.libats.data.DataType.HashMethod import com.advancedtelematic.libtuf.data.ClientDataType.ClientTargetItem -import com.advancedtelematic.libtuf.data.TufDataType.TargetFilename +import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, TargetFilename} import com.advancedtelematic.libtuf_server.crypto.Sha256Digest import eu.timepit.refined.api.Refined import io.circe.Json +import com.advancedtelematic.libats.codecs.CirceRefined.* class RepoTargetsResourceSpec extends TufReposerverSpec @@ -33,6 +30,10 @@ class RepoTargetsResourceSpec The library will endure; it is the universe. As for us, everything has not been written; we are not turning into phantoms. We walk the corridors, searching the shelves and rearranging them, looking for lines of meaning amid leagues of cacophony and incoherence, reading the history of the past and our future, collecting our thoughts and collecting the thoughts of others, and every so often glimpsing mirrors, in which we may recognize creatures of the information.” """.stripMargin)) + val testEntity2 = HttpEntity(ByteString(""" + If honor and wisdom and happiness are not for me, let them be for others. Let heaven exist, though my place be in hell + """.stripMargin)) + testWithRepo("GET returns delegation items ") { implicit ns => implicit repoId => addTargetToRepo(repoId) @@ -322,13 +323,21 @@ class RepoTargetsResourceSpec status shouldBe StatusCodes.NoContent } - Get(apiUriV2(s"user_repo/search?hashes=352ce6b496cece167046d00d8a6431ffa43646b378cce4e3013d1d9aeef8dbb4,a1fb50e6c86fae1679ef3351296fd6713411a08cf8dd1790a4fd05fae8688161")).namespaced ~> routes ~> check { + Get( + apiUriV2( + s"user_repo/search?hashes=352ce6b496cece167046d00d8a6431ffa43646b378cce4e3013d1d9aeef8dbb4,a1fb50e6c86fae1679ef3351296fd6713411a08cf8dd1790a4fd05fae8688161" + ) + ).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK val values = responseAs[PaginationResult[Package]].values values should have size 1 } - Get(apiUriV2(s"user_repo/search?hashes=0000000000000000000000000000000000000000000000000000000000000000")).namespaced ~> routes ~> check { + Get( + apiUriV2( + s"user_repo/search?hashes=0000000000000000000000000000000000000000000000000000000000000000" + ) + ).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK val values = responseAs[PaginationResult[Package]].values values shouldBe empty @@ -527,13 +536,21 @@ class RepoTargetsResourceSpec values should have size 1 } - Get(apiUriV2(s"user_repo/grouped-search?hashes=0000000000000000000000000000000000000000000000000000000000000000")).namespaced ~> routes ~> check { + Get( + apiUriV2( + s"user_repo/grouped-search?hashes=0000000000000000000000000000000000000000000000000000000000000000" + ) + ).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK val values = responseAs[PaginationResult[AggregatedPackage]].values values shouldBe empty } - Get(apiUriV2(s"user_repo/grouped-search?hashes=352ce6b496cece167046d00d8a6431ffa43646b378cce4e3013d1d9aeef8dbb4")).namespaced ~> routes ~> check { + Get( + apiUriV2( + s"user_repo/grouped-search?hashes=352ce6b496cece167046d00d8a6431ffa43646b378cce4e3013d1d9aeef8dbb4" + ) + ).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK val values = responseAs[PaginationResult[AggregatedPackage]].values values should have size 1 @@ -576,4 +593,31 @@ class RepoTargetsResourceSpec } } + testWithRepo("GET hardwareids-packages returns hwids for which there are packages") { implicit ns => implicit repoId => + addTargetToRepo(repoId) + + Put( + apiUri("user_repo/targets/mypkg_file?name=library&version=0.0.1&hardwareIds=myid001,myid002"), + testEntity + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.NoContent + } + + Put( + apiUri("user_repo/targets/mypkg_file2?name=library&version=0.0.2&hardwareIds=myid002,myid003"), + testEntity2 + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.NoContent + } + + Get(apiUriV2(s"user_repo/hardwareids-packages")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val result = responseAs[PaginationResult[HardwareIdentifier]] + result.total shouldBe 3 + result.offset shouldBe 0 + result.limit shouldBe 3 + result.values.map(_.value) should contain theSameElementsAs List("myid001", "myid002", "myid003") + } + } + } From ff766209d06e2e9924cfad056c3cb3e4ac9dda7a Mon Sep 17 00:00:00 2001 From: Ben Clouser Date: Wed, 4 Sep 2024 10:23:21 -0400 Subject: [PATCH 3/3] Update mariadb in github workflow 10.4 -> 10.11 Signed-off-by: Ben Clouser --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f5f2a6..352050ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: image: uptane/tuf-nginx:latest db: - image: mariadb:10.4 + image: mariadb:10.11 env: MYSQL_ROOT_PASSWORD: "root" MYSQL_DATABASE: "ota_tuf"