From 9cf521e1e660339761e648595d792f6ec5ff3e4f Mon Sep 17 00:00:00 2001 From: Daniel K Date: Tue, 31 May 2022 15:13:39 +0200 Subject: [PATCH] Feature/1693 api v3 disabled fails validation (#2066) * #1693 As `disabled` flag cases to fail validation: - custom isDisabledCheck is not needed anymore (V3 services use `validate` for this) - `EntityDisabledException` removed, replaced by a `Validation` with `disabled` key. - validation fixed for Schemas (did not include super-check) - V3 integTests updated to cover the `disabled` = validation fail * #1693 disable fail due to nonEmpty used in now carries a wrapper with an error message (`UsedIn` wrapped in `EntityInUseException`) * #1693 `Future {throw x}` replaced with `Future.failed(x)` in rest_api, cleanup --- .../controllers/RestExceptionHandler.scala | 5 - .../VersionedModelController.scala | 4 +- .../v3/VersionedModelControllerV3.scala | 33 ++--- .../exceptions/EntityDisabledException.scala | 19 --- .../rest_api/services/AttachmentService.scala | 6 +- .../rest_api/services/DatasetService.scala | 6 +- .../services/MappingTableService.scala | 6 +- .../rest_api/services/ModelService.scala | 4 +- .../rest_api/services/MonitoringService.scala | 7 +- .../services/PropertyDefinitionService.scala | 7 +- .../rest_api/services/RunService.scala | 6 +- .../rest_api/services/SchemaService.scala | 6 +- .../services/VersionedModelService.scala | 82 +++-------- .../services/v3/DatasetServiceV3.scala | 2 +- .../services/v3/MappingTableServiceV3.scala | 3 +- .../v3/PropertyDefinitionServiceV3.scala | 2 +- .../services/v3/SchemaServiceV3.scala | 20 ++- .../services/v3/VersionedModelServiceV3.scala | 111 +++++++++++++++ .../DatasetApiIntegrationSuite.scala | 77 +++++++++- .../DatasetControllerV3IntegrationSuite.scala | 133 +++++++++++++++--- ...ingTableControllerV3IntegrationSuite.scala | 44 ++++-- ...finitionControllerV3IntegrationSuite.scala | 28 +++- .../SchemaControllerV3IntegrationSuite.scala | 72 +++++++++- 23 files changed, 502 insertions(+), 181 deletions(-) delete mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala create mode 100644 rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/VersionedModelServiceV3.scala diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala index 115c18469..3548859be 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala @@ -67,11 +67,6 @@ class RestExceptionHandler { ResponseEntity.status(HttpStatus.I_AM_A_TEAPOT).build[Any]() // Could change for LOCKED but I like this more } - @ExceptionHandler(value = Array(classOf[EntityDisabledException])) - def handleEntityDisabled(exception: EntityDisabledException): ResponseEntity[EntityDisabledException] = { - ResponseEntity.status(HttpStatus.BAD_REQUEST).body(exception) - } - @ExceptionHandler(value = Array(classOf[SchemaParsingException])) def handleBadRequestException(exception: SchemaParsingException): ResponseEntity[Any] = { val response = RestResponse(exception.message, Option(SchemaParsingError.fromException(exception))) diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala index af0c844f0..a8b25598e 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala @@ -115,8 +115,8 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au @ResponseStatus(HttpStatus.CREATED) def importSingleEntity(@AuthenticationPrincipal principal: UserDetails, @RequestBody importObject: ExportableObject[C]): CompletableFuture[C] = { - versionedModelService.importSingleItemV2(importObject.item, principal.getUsername, importObject.metadata).map { - case Some(entity) => entity + versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { + case Some((entity, validation)) => entity // validation is disregarded for V2, import-v2 has its own case None => throw notFound() } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala index 911604609..2d1e7fcdc 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/VersionedModelControllerV3.scala @@ -15,7 +15,6 @@ package za.co.absa.enceladus.rest_api.controllers.v3 -import com.mongodb.client.result.UpdateResult import org.springframework.http.{HttpStatus, ResponseEntity} import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.core.userdetails.UserDetails @@ -26,9 +25,9 @@ import za.co.absa.enceladus.model.versionedModel._ import za.co.absa.enceladus.model.{ExportableObject, UsedIn, Validation} import za.co.absa.enceladus.rest_api.controllers.BaseController import za.co.absa.enceladus.rest_api.controllers.v3.VersionedModelControllerV3.LatestVersionKey -import za.co.absa.enceladus.rest_api.exceptions.{EntityDisabledException, NotFoundException, ValidationException} +import za.co.absa.enceladus.rest_api.exceptions.NotFoundException import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload -import za.co.absa.enceladus.rest_api.services.VersionedModelService +import za.co.absa.enceladus.rest_api.services.v3.VersionedModelServiceV3 import java.net.URI import java.util.Optional @@ -42,7 +41,7 @@ object VersionedModelControllerV3 { } abstract class VersionedModelControllerV3[C <: VersionedModel with Product - with Auditable[C]](versionedModelService: VersionedModelService[C]) extends BaseController { + with Auditable[C]](versionedModelService: VersionedModelServiceV3[C]) extends BaseController { import za.co.absa.enceladus.rest_api.utils.implicits._ @@ -110,7 +109,7 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product if (name != importObject.item.name) { Future.failed(new IllegalArgumentException(s"URL and payload entity name mismatch: '$name' != '${importObject.item.name}'")) } else { - versionedModelService.importSingleItemV3(importObject.item, principal.getUsername, importObject.metadata).map { + versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map { case Some((entity, validation)) => // stripping two last segments, instead of /api-v3/dastasets/dsName/import + /dsName/dsVersion we want /api-v3/dastasets + /dsName/dsVersion createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 2).body(validation) @@ -131,13 +130,10 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product def create(@AuthenticationPrincipal principal: UserDetails, @RequestBody item: C, request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = { - versionedModelService.isDisabled(item.name).flatMap { isDisabled => - if (isDisabled) { - Future.failed(EntityDisabledException(s"Entity ${item.name} is disabled. Enable it first (PUT) to push new versions (PUT).")) - } else { + + // enabled check is part of the validation versionedModelService.create(item, principal.getUsername) - } - }.map { + .map { case Some((entity, validation)) => createdWithNameVersionLocationBuilder(entity.name, entity.version, request).body(validation) case None => throw notFound() } @@ -156,16 +152,11 @@ abstract class VersionedModelControllerV3[C <: VersionedModel with Product } else if (version != item.version) { Future.failed(new IllegalArgumentException(s"URL and payload version mismatch: ${version} != ${item.version}")) } else { - versionedModelService.isDisabled(item.name).flatMap { isDisabled => - if (isDisabled) { - throw EntityDisabledException(s"Entity ${item.name} is disabled. Enable it first to create new versions.") - } else { - versionedModelService.update(user.getUsername, item).map { - case Some((updatedEntity, validation)) => - createdWithNameVersionLocationBuilder(updatedEntity.name, updatedEntity.version, request, stripLastSegments = 2).body(validation) - case None => throw notFound() - } - } + // disable check is already part of V3 validation + versionedModelService.update(user.getUsername, item).map { + case Some((updatedEntity, validation)) => + createdWithNameVersionLocationBuilder(updatedEntity.name, updatedEntity.version, request, stripLastSegments = 2).body(validation) + case None => throw notFound() } } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala deleted file mode 100644 index 54a770d9d..000000000 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/exceptions/EntityDisabledException.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2018 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.enceladus.rest_api.exceptions - -case class EntityDisabledException(message:String = "", cause: Throwable = None.orNull) extends Exception(message, cause) - diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/AttachmentService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/AttachmentService.scala index 4a49d919c..5676167a6 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/AttachmentService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/AttachmentService.scala @@ -25,11 +25,13 @@ import scala.concurrent.Future import za.co.absa.enceladus.rest_api.exceptions.NotFoundException @Service -class AttachmentService @Autowired()(attachmentMongoRepository: AttachmentMongoRepository, +class AttachmentService @Autowired()(val mongoRepository: AttachmentMongoRepository, schemaMongoRepository: SchemaMongoRepository, datasetMongoRepository: DatasetMongoRepository, mappingTableMongoRepository: MappingTableMongoRepository) - extends ModelService(attachmentMongoRepository) { + extends ModelService[MenasAttachment] { + + protected val attachmentMongoRepository: AttachmentMongoRepository = mongoRepository // alias import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala index 6a9028895..93f226c6d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/DatasetService.scala @@ -36,10 +36,12 @@ import scala.util.{Failure, Success} @Service -class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository, +class DatasetService @Autowired()(val mongoRepository: DatasetMongoRepository, oozieRepository: OozieRepository, propertyDefinitionService: PropertyDefinitionService) - extends VersionedModelService(datasetMongoRepository) { + extends VersionedModelService[Dataset] { + + protected val datasetMongoRepository: DatasetMongoRepository = mongoRepository // alias import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala index 9bcecfe86..fb5e6026a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MappingTableService.scala @@ -23,8 +23,10 @@ import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, Mappi import scala.concurrent.Future @Service -class MappingTableService @Autowired() (mappingTableMongoRepository: MappingTableMongoRepository, - datasetMongoRepository: DatasetMongoRepository) extends VersionedModelService(mappingTableMongoRepository) { +class MappingTableService @Autowired() (val mongoRepository: MappingTableMongoRepository, + datasetMongoRepository: DatasetMongoRepository) extends VersionedModelService[MappingTable] { + + protected val mappingTableMongoRepository: MappingTableMongoRepository = mongoRepository // alias import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/ModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/ModelService.scala index 427daf402..7549158d3 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/ModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/ModelService.scala @@ -20,7 +20,9 @@ import za.co.absa.enceladus.rest_api.repositories.MongoRepository import scala.concurrent.Future -abstract class ModelService[C](mongoRepository: MongoRepository[C]) { +trait ModelService[C] { + + def mongoRepository: MongoRepository[C] import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MonitoringService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MonitoringService.scala index 5c946e396..2819f5957 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MonitoringService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/MonitoringService.scala @@ -17,18 +17,19 @@ package za.co.absa.enceladus.rest_api.services import org.springframework.beans.factory.annotation.{Autowired, Value} import org.springframework.stereotype.Service +import za.co.absa.enceladus.model.Run import za.co.absa.enceladus.rest_api.repositories.MonitoringMongoRepository import scala.concurrent.Future @Service -class MonitoringService @Autowired()(monitoringMongoRepository: MonitoringMongoRepository) - extends ModelService(monitoringMongoRepository) { +class MonitoringService @Autowired()(val mongoRepository: MonitoringMongoRepository) + extends ModelService[Run] { import scala.concurrent.ExecutionContext.Implicits.global def getMonitoringDataPoints(datasetName: String, startDate: String, endDate: String): Future[String] = { - monitoringMongoRepository.getMonitoringDataPoints(datasetName, startDate, endDate).map(_.mkString("[", ",", "]")) + mongoRepository.getMonitoringDataPoints(datasetName, startDate, endDate).map(_.mkString("[", ",", "]")) } } diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala index cb5bc9b08..b11169d0c 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/PropertyDefinitionService.scala @@ -24,9 +24,10 @@ import za.co.absa.enceladus.model.properties.PropertyDefinition import scala.concurrent.Future @Service("propertyDefinitionService") // by-name qualifier: V2 implementations use the base implementation, not v3 -class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: PropertyDefinitionMongoRepository) - extends VersionedModelService(propertyDefMongoRepository) { +class PropertyDefinitionService @Autowired()(val mongoRepository: PropertyDefinitionMongoRepository) + extends VersionedModelService[PropertyDefinition] { + protected val propertyDefMongoRepository: PropertyDefinitionMongoRepository = mongoRepository // alias import scala.concurrent.ExecutionContext.Implicits.global override def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] = { @@ -44,7 +45,7 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert } def getDistinctCount(): Future[Int] = { - propertyDefMongoRepository.distinctCount() + mongoRepository.distinctCount() } override def create(newPropertyDef: PropertyDefinition, username: String): Future[Option[(PropertyDefinition, Validation)]] = { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/RunService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/RunService.scala index af83ae16a..f4ed05ff2 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/RunService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/RunService.scala @@ -31,8 +31,10 @@ import scala.concurrent.Future import scala.util.{Failure, Success, Try} @Service -class RunService @Autowired()(runMongoRepository: RunMongoRepository) - extends ModelService(runMongoRepository) { +class RunService @Autowired()(val mongoRepository: RunMongoRepository) + extends ModelService[Run] { + + protected val runMongoRepository: RunMongoRepository = mongoRepository // alias def getRunSummariesPerDatasetName(): Future[Seq[RunDatasetNameGroupedSummary]] = { runMongoRepository.getRunSummariesPerDatasetName() diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala index a4f5eb763..b494d267d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/SchemaService.scala @@ -25,10 +25,12 @@ import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor import scala.concurrent.Future @Service -class SchemaService @Autowired() (schemaMongoRepository: SchemaMongoRepository, +class SchemaService @Autowired() (val mongoRepository: SchemaMongoRepository, mappingTableMongoRepository: MappingTableMongoRepository, datasetMongoRepository: DatasetMongoRepository, - sparkMenasConvertor: SparkMenasSchemaConvertor) extends VersionedModelService(schemaMongoRepository) { + sparkMenasConvertor: SparkMenasSchemaConvertor) extends VersionedModelService[Schema] { + + protected val schemaMongoRepository: SchemaMongoRepository = mongoRepository // alias import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala index de26f6e36..e96849d61 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/VersionedModelService.scala @@ -20,7 +20,6 @@ import org.slf4j.LoggerFactory import org.springframework.security.core.context.SecurityContextHolder import org.springframework.security.core.userdetails.UserDetails import za.co.absa.enceladus.model.{ModelVersion, Schema, UsedIn, Validation} -import za.co.absa.enceladus.model.menas._ import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} import za.co.absa.enceladus.rest_api.exceptions._ import za.co.absa.enceladus.rest_api.repositories.VersionedMongoRepository @@ -31,53 +30,51 @@ import com.mongodb.MongoWriteException import VersionedModelService._ // scalastyle:off number.of.methods -abstract class VersionedModelService[C <: VersionedModel with Product with Auditable[C]] -(versionedMongoRepository: VersionedMongoRepository[C]) extends ModelService(versionedMongoRepository) { +trait VersionedModelService[C <: VersionedModel with Product with Auditable[C]] + extends ModelService[C] { + + def mongoRepository: VersionedMongoRepository[C] import scala.concurrent.ExecutionContext.Implicits.global private[services] val logger = LoggerFactory.getLogger(this.getClass) def getLatestVersionsSummarySearch(searchQuery: Option[String]): Future[Seq[VersionedSummary]] = { - versionedMongoRepository.getLatestVersionsSummarySearch(searchQuery) + mongoRepository.getLatestVersionsSummarySearch(searchQuery) } def getLatestVersions(): Future[Seq[C]] = { - versionedMongoRepository.getLatestVersions(None) + mongoRepository.getLatestVersions(None) } def getSearchSuggestions(): Future[Seq[String]] = { - versionedMongoRepository.getDistinctNamesEnabled() + mongoRepository.getDistinctNamesEnabled() } def getVersion(name: String, version: Int): Future[Option[C]] = { - versionedMongoRepository.getVersion(name, version) + mongoRepository.getVersion(name, version) } def getAllVersions(name: String): Future[Seq[C]] = { - versionedMongoRepository.getAllVersions(name) + mongoRepository.getAllVersions(name) } def getLatestVersion(name: String): Future[Option[C]] = { - versionedMongoRepository.getLatestVersionValue(name).flatMap({ + mongoRepository.getLatestVersionValue(name).flatMap({ case Some(version) => getVersion(name, version) case _ => throw NotFoundException() }) } def getLatestVersionNumber(name: String): Future[Int] = { - versionedMongoRepository.getLatestVersionValue(name).flatMap({ + mongoRepository.getLatestVersionValue(name).flatMap({ case Some(version) => Future(version) case _ => throw NotFoundException() }) } def getLatestVersionValue(name: String): Future[Option[Int]] = { - versionedMongoRepository.getLatestVersionValue(name) - } - - def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { - versionedMongoRepository.getLatestVersionSummary(name) + mongoRepository.getLatestVersionValue(name) } def exportSingleItem(name: String, version: Int): Future[String] = { @@ -95,7 +92,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit } // v2 has external validate validation applied only to imports (not create/edits) via validateSingleImport - def importSingleItemV2(item: C, username: String, metadata: Map[String, String]): Future[Option[C]] = { + def importSingleItem(item: C, username: String, metadata: Map[String, String]): Future[Option[(C, Validation)]] = { for { validation <- validateSingleImport(item, metadata) result <- { @@ -105,12 +102,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit throw ValidationException(validation) } } - } yield result.map(_._1) // v disregards internal common update-based validation - } - - // v3 has internal validation on importItem (because it is based on update - def importSingleItemV3(item: C, username: String, metadata: Map[String, String]): Future[Option[(C, Validation)]] = { - importItem(item, username) + } yield result } private[services] def validateSingleImport(item: C, metadata: Map[String, String]): Future[Validation] = { @@ -152,7 +144,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit for { versions <- { //store all in version ascending order - val all = versionedMongoRepository.getAllVersions(name, inclDisabled = true).map(_.sortBy(_.version)) + val all = mongoRepository.getAllVersions(name, inclDisabled = true).map(_.sortBy(_.version)) //get those relevant to us if (fromVersion.isDefined) { all.map(_.filter(_.version <= fromVersion.get)) @@ -197,10 +189,6 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit def getUsedIn(name: String, version: Option[Int]): Future[UsedIn] - private[rest_api] def getMenasRef(item: C): MenasReference = { - MenasReference(Some(versionedMongoRepository.collectionBaseName), item.name, item.version) - } - private[rest_api] def create(item: C, username: String): Future[Option[(C, Validation)]] = { // individual validations are deliberately not run in parallel - the latter may not be needed if the former fails for { @@ -209,7 +197,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit creationValidation <- validateForCreation(item) } yield generalValidation.merge(creationValidation) _ <- if (validation.isValid) { - versionedMongoRepository.create(item, username) + mongoRepository.create(item, username) .recover { case e: MongoWriteException => throw ValidationException(Validation().withError("name", s"entity with name already exists: '${item.name}'")) @@ -247,7 +235,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit throw ValidationException(validation) } } - update <- versionedMongoRepository.update(username, transformed) + update <- mongoRepository.update(username, transformed) .recover { case e: MongoWriteException => throw ValidationException(Validation().withError("version", s"entity '$itemName' with this version already exists: ${itemVersion + 1}")) @@ -263,21 +251,6 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit } } - def findRefEqual(refNameCol: String, refVersionCol: String, name: String, version: Option[Int]): Future[Seq[MenasReference]] = { - versionedMongoRepository.findRefEqual(refNameCol, refVersionCol, name, version) - } - - /** - * Enables all versions of the entity by name. - * @param name - */ - def enableEntity(name: String): Future[UpdateResult] = { - val auth = SecurityContextHolder.getContext.getAuthentication - val principal = auth.getPrincipal.asInstanceOf[UserDetails] - - versionedMongoRepository.enableAllVersions(name, principal.getUsername) - } - def disableVersion(name: String, version: Option[Int]): Future[UpdateResult] = { val auth = SecurityContextHolder.getContext.getAuthentication val principal = auth.getPrincipal.asInstanceOf[UserDetails] @@ -292,29 +265,12 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit val entityVersionStr = s"""entity "$name"${ version.map(" v" + _).getOrElse("")}""" // either "entity MyName" or "entity MyName v23" throw EntityInUseException(s"""Cannot disable $entityVersionStr, because it is used in the following entities""", usedIn) } else { - versionedMongoRepository.disableVersion(name, version, principal.getUsername) + mongoRepository.disableVersion(name, version, principal.getUsername) } } def isDisabled(name: String): Future[Boolean] = { - versionedMongoRepository.isDisabled(name) - } - - /** - * Retrieves model@version and calls - * [[za.co.absa.enceladus.rest_api.services.VersionedModelService#validate(java.lang.Object)]] - * - * In order to extend this behavior, override the mentioned method instead. (that's why this is `final`) - * - * @param name - * @param version - * @return - */ - final def validate(name: String, version: Int): Future[Validation] = { - getVersion(name, version).flatMap({ - case Some(entity) => validate(entity) - case _ => Future.failed(NotFoundException(s"Entity by name=$name, version=$version is not found!")) - }) + mongoRepository.isDisabled(name) } /** diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala index afd647010..d454a6f6d 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/DatasetServiceV3.scala @@ -34,7 +34,7 @@ class DatasetServiceV3 @Autowired()(datasetMongoRepository: DatasetMongoReposito mappingTableService: MappingTableServiceV3, val schemaService: SchemaServiceV3) extends DatasetService(datasetMongoRepository, oozieRepository, propertyDefinitionService) - with HavingSchemaService { + with HavingSchemaService with VersionedModelServiceV3[Dataset] { import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala index 12fb346f8..f9858c7cc 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/MappingTableServiceV3.scala @@ -27,7 +27,8 @@ import scala.concurrent.Future class MappingTableServiceV3 @Autowired()(mappingTableMongoRepository: MappingTableMongoRepository, datasetMongoRepository: DatasetMongoRepository, val schemaService: SchemaServiceV3) - extends MappingTableService(mappingTableMongoRepository, datasetMongoRepository) with HavingSchemaService { + extends MappingTableService(mappingTableMongoRepository, datasetMongoRepository) with HavingSchemaService + with VersionedModelServiceV3[MappingTable] { import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala index bab32b4df..6ce31a490 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/PropertyDefinitionServiceV3.scala @@ -27,7 +27,7 @@ import scala.concurrent.Future @Service class PropertyDefinitionServiceV3 @Autowired()(propertyDefMongoRepository: PropertyDefinitionMongoRepository, datasetMongoRepository: DatasetMongoRepository) - extends PropertyDefinitionService(propertyDefMongoRepository) { + extends PropertyDefinitionService(propertyDefMongoRepository) with VersionedModelServiceV3[PropertyDefinition] { import scala.concurrent.ExecutionContext.Implicits.global diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala index 7878fc7c7..b9dd0595a 100644 --- a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/SchemaServiceV3.scala @@ -15,15 +15,12 @@ package za.co.absa.enceladus.rest_api.services.v3 -import org.apache.spark.sql.types.StructType import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.model.{Schema, UsedIn, Validation} +import za.co.absa.enceladus.model.{Schema, SchemaField, UsedIn, Validation} import za.co.absa.enceladus.rest_api.repositories.{DatasetMongoRepository, MappingTableMongoRepository, SchemaMongoRepository} -import za.co.absa.enceladus.rest_api.services.{SchemaService, VersionedModelService} +import za.co.absa.enceladus.rest_api.services.SchemaService import za.co.absa.enceladus.rest_api.utils.converters.SparkMenasSchemaConvertor -import scala.concurrent.ExecutionContext.Implicits.global - import scala.concurrent.Future @@ -32,12 +29,21 @@ class SchemaServiceV3 @Autowired()(schemaMongoRepository: SchemaMongoRepository, mappingTableMongoRepository: MappingTableMongoRepository, datasetMongoRepository: DatasetMongoRepository, sparkMenasConvertor: SparkMenasSchemaConvertor) - extends SchemaService(schemaMongoRepository, mappingTableMongoRepository, datasetMongoRepository, sparkMenasConvertor) { + extends SchemaService(schemaMongoRepository, mappingTableMongoRepository, datasetMongoRepository, sparkMenasConvertor) + with VersionedModelServiceV3[Schema]{ import scala.concurrent.ExecutionContext.Implicits.global override def validate(item: Schema): Future[Validation] = { - if (item.fields.isEmpty) { + for { + originalValidation <- super.validate(item) + fieldsValidation <- validateSchemaFields(item.fields) + } yield originalValidation.merge(fieldsValidation) + + } + + protected def validateSchemaFields(fields: Seq[SchemaField]): Future[Validation] = { + if (fields.isEmpty) { // V3 disallows empty schema fields - V2 allowed it at first that to get updated by an attachment upload/remote-load Future.successful(Validation.empty.withError("schema-fields","No fields found! There must be fields defined for actual usage.")) } else { diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/VersionedModelServiceV3.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/VersionedModelServiceV3.scala new file mode 100644 index 000000000..a053db392 --- /dev/null +++ b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/VersionedModelServiceV3.scala @@ -0,0 +1,111 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.rest_api.services.v3 + +import org.mongodb.scala.result.UpdateResult +import org.slf4j.LoggerFactory +import org.springframework.security.core.context.SecurityContextHolder +import org.springframework.security.core.userdetails.UserDetails +import za.co.absa.enceladus.model.{UsedIn, Validation} +import za.co.absa.enceladus.model.menas.audit._ +import za.co.absa.enceladus.model.versionedModel.{VersionedModel, VersionedSummary} +import za.co.absa.enceladus.rest_api.exceptions._ +import za.co.absa.enceladus.rest_api.services.VersionedModelService + +import scala.concurrent.Future + +// scalastyle:off number.of.methods +trait VersionedModelServiceV3[C <: VersionedModel with Product with Auditable[C]] extends VersionedModelService[C] { + + import scala.concurrent.ExecutionContext.Implicits.global + + override private[services] val logger = LoggerFactory.getLogger(this.getClass) + + def getLatestVersionSummary(name: String): Future[Option[VersionedSummary]] = { + mongoRepository.getLatestVersionSummary(name) + } + + // v3 has internal validation on importItem (because it is based on update), v2 only had it on import + override def importSingleItem(item: C, username: String, metadata: Map[String, String]): Future[Option[(C, Validation)]] = { + val metadataValidation = validateMetadata(metadata) + if (metadataValidation.isValid) { + // even if valid, merge validations results (warnings may be theoretically present) + importItem(item, username).map(_.map { case (item, validation) => (item, validation.merge(metadataValidation)) }) + } else { + Future.failed(ValidationException(metadataValidation)) + } + } + + override private[services] def update(username: String, itemName: String, itemVersion: Int) + (transform: C => C): Future[Option[(C, Validation)]] = { + this.updateFuture(username, itemName, itemVersion) { item: C => + for { + _ <- validate(item) // V3 validates the to-be-updated item, too (V2 does not) + updatedItem <- Future { + transform(item) + } + } yield updatedItem + } + } + + /** + * Enables all versions of the entity by name. + * @param name + */ + def enableEntity(name: String): Future[UpdateResult] = { + val auth = SecurityContextHolder.getContext.getAuthentication + val principal = auth.getPrincipal.asInstanceOf[UserDetails] + + // todo check validation of used-in dependees? #2065 + + mongoRepository.enableAllVersions(name, principal.getUsername) + } + + + /** + * Retrieves model@version and calls + * [[validate(java.lang.Object)]] + * + * In order to extend this behavior, override the mentioned method instead. (that's why this is `final`) + */ + final def validate(name: String, version: Int): Future[Validation] = { + getVersion(name, version).flatMap({ + case Some(entity) => validate(entity) + case _ => Future.failed(NotFoundException(s"Entity by name=$name, version=$version is not found!")) + }) + } + + /** + * Validates the item as V2 + checks that it is enabled + */ + override def validate(item: C): Future[Validation] = { + for { + superValidation <- super.validate(item) + enabledValidation <- validateEnabled(item) + } yield superValidation.merge(enabledValidation) + } + + protected[services] def validateEnabled(item: C): Future[Validation] = { + if (item.disabled) { + Future.successful(Validation.empty.withError("disabled", s"Entity ${item.name} is disabled!")) + } else { + Future.successful(Validation.empty) + } + } + +} + + diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala index 4f13250da..dbdbb13cd 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/DatasetApiIntegrationSuite.scala @@ -17,6 +17,7 @@ package za.co.absa.enceladus.rest_api.integration.controllers import org.junit.runner.RunWith import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @@ -27,14 +28,14 @@ import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, PropertyType, StringPropertyType} -import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory} +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.rest_api.integration.fixtures._ @RunWith(classOf[SpringRunner]) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(Array("withEmbeddedMongo")) -class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAll { +class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAll with Matchers { @Autowired private val datasetFixture: DatasetFixtureService = null @@ -42,10 +43,13 @@ class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAl @Autowired private val propertyDefinitionFixture: PropertyDefinitionFixtureService = null + @Autowired + private val schemaFixture: SchemaFixtureService = null + private val apiUrl = "/dataset" // fixtures are cleared after each test - override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture) + override def fixtures: List[FixtureService[_]] = List(datasetFixture, propertyDefinitionFixture, schemaFixture) s"POST $apiUrl/create" can { "return 201" when { @@ -177,7 +181,74 @@ class DatasetApiIntegrationSuite extends BaseRestApiTestV2 with BeforeAndAfterAl } } } + } + + private def importableDs(name: String = "dataset", metadataContent: String = """"exportVersion":1"""): String = { + s"""{"metadata":{$metadataContent},"item":{ + |"name":"$name", + |"hdfsPath":"/dummy/path", + |"hdfsPublishPath":"/dummy/publish/path", + |"schemaName":"dummySchema", + |"schemaVersion":1, + |"conformance":[], + |"properties":{"key2":"val2","key1":"val1"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "") + } + + s"POST $apiUrl/importItem" should { + "return 201" when { + "the import is successful" should { + "return the imported PD representation" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // referenced schema must exists + + val response = sendPost[String, Dataset](s"$apiUrl/importItem", bodyOpt = Some(importableDs())) + assertCreated(response) + + val actual = response.getBody + val expectedDs = DatasetFactory.getDummyDataset("dataset", properties = Some(Map("key1" -> "val1", "key2" -> "val2"))) + val expected = toExpected(expectedDs, actual) + assert(actual == expected) + } + } + } + "return 404" when { + "referenced schema does not exits" in { + // referenced schema missing + val response = sendPost[String, Validation](s"$apiUrl/importItem", bodyOpt = Some(importableDs())) + assertBadRequest(response) + + response.getBody shouldBe + Validation.empty.withError("item.schema", "schema dummySchema v1 defined for the dataset could not be found") + } + "imported dataset has disallowed name" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // referenced schema must exists + + val response = sendPost[String, Validation](s"$apiUrl/importItem", bodyOpt = Some(importableDs(name = "invalid %$. name"))) + assertBadRequest(response) + + response.getBody shouldBe Validation.empty.withError("item.name", "name 'invalid %$. name' contains unsupported characters") + } + "imported dataset has unsupported metadata exportVersion" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // referenced schema must exists + + val response = sendPost[String, Validation](s"$apiUrl/importItem", + bodyOpt = Some(importableDs(metadataContent = """"exportVersion":6""" ))) + assertBadRequest(response) + + response.getBody shouldBe Validation.empty.withError("metadata.exportApiVersion", + "Export/Import API version mismatch. Acceptable version is 1. Version passed is 6") + } + "imported dataset has unsupported metadata" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // referenced schema must exists + val response = sendPost[String, Validation](s"$apiUrl/importItem", + bodyOpt = Some(importableDs(metadataContent = """"otherMetaKey":"otherMetaVal"""" ))) + assertBadRequest(response) + + response.getBody shouldBe Validation.empty.withError("metadata.exportApiVersion", + "Export/Import API version mismatch. Acceptable version is 1. Version passed is null") + } + } } // Dataset specific: diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala index a036eb426..436512e53 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/DatasetControllerV3IntegrationSuite.scala @@ -30,7 +30,6 @@ import za.co.absa.enceladus.model.properties.propertyType.EnumPropertyType import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} import za.co.absa.enceladus.model.versionedModel.NamedVersion import za.co.absa.enceladus.model.{Dataset, UsedIn, Validation} -import za.co.absa.enceladus.rest_api.exceptions.EntityDisabledException import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ import za.co.absa.enceladus.rest_api.models.rest.DisabledPayload @@ -117,13 +116,13 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(dataset1) val dataset2 = DatasetFactory.getDummyDataset("dummyDs", description = Some("a new version attempt")) - val response = sendPost[Dataset, EntityDisabledException](apiUrl, bodyOpt = Some(dataset2)) + val response = sendPost[Dataset, Validation](apiUrl, bodyOpt = Some(dataset2)) assertBadRequest(response) - response.getBody.getMessage should include("Entity dummyDs is disabled. Enable it first") + response.getBody shouldBe Validation.empty.withError("name", "entity with name already exists: 'dummyDs'") + // even if disabled, the dulicate name check has precedence and is reported first } } - } s"GET $apiUrl/{name}" should { @@ -353,10 +352,10 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA datasetFixture.add(dataset1) val dataset2 = DatasetFactory.getDummyDataset("dummyDs", description = Some("ds update")) - val response = sendPut[Dataset, EntityDisabledException](s"$apiUrl/dummyDs/1", bodyOpt = Some(dataset2)) + val response = sendPut[Dataset, Validation](s"$apiUrl/dummyDs/1", bodyOpt = Some(dataset2)) assertBadRequest(response) - response.getBody.getMessage should include("Entity dummyDs is disabled. Enable it first") + response.getBody shouldBe Validation.empty.withError("disabled", "Entity dummyDs is disabled!") } } } @@ -401,36 +400,87 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } s"POST $apiUrl/{name}/import" should { - val importableDs = - """{"metadata":{"exportVersion":1},"item":{ - |"name":"datasetXYZ", - |"description":"Hi, I am the import", - |"hdfsPath":"/dummy/path", - |"hdfsPublishPath":"/dummy/publish/path", - |"schemaName":"dummySchema", - |"schemaVersion":1, - |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], - |"properties":{"key2":"val2","key1":"val1"} - |}}""".stripMargin.replaceAll("[\\r\\n]", "") + def importableDs(name: String = "datasetXYZ", metadataContent: String = """"exportVersion":1"""): String = { + s"""{"metadata":{$metadataContent},"item":{ + |"name":"$name", + |"description":"Hi, I am the import", + |"hdfsPath":"/dummy/path", + |"hdfsPublishPath":"/dummy/publish/path", + |"schemaName":"dummySchema", + |"schemaVersion":1, + |"conformance":[{"_t":"LiteralConformanceRule","order":0,"outputColumn":"outputCol1","controlCheckpoint":false,"value":"litValue1"}], + |"properties":{"key2":"val2","key1":"val1"} + |}}""".stripMargin.replaceAll("[\\r\\n]", "") + } "return 400" when { "a Dataset with the given name" should { "fail when name in the URL and payload is mismatched" in { val response = sendPost[String, String](s"$apiUrl/datasetABC/import", - bodyOpt = Some(importableDs)) + bodyOpt = Some(importableDs())) response.getStatusCode shouldBe HttpStatus.BAD_REQUEST response.getBody should include("name mismatch: 'datasetABC' != 'datasetXYZ'") } } - "imported Dataset fails validation" in { + "imported Dataset fails validation 1 (ref'd entities)" in { schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("key1")) // key2 propdef is missing - val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs())) response.getStatusCode shouldBe HttpStatus.BAD_REQUEST response.getBody shouldBe Validation.empty.withError("key2", "There is no property definition for key 'key2'.") } + "imported Dataset fails validation 2 (entityName invalid)" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("key1")) // key2 propdef is missing + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key2") + ) + + val response = sendPost[String, Validation](s"$apiUrl/name$${*/import", bodyOpt = Some(importableDs(name = "name${*"))) + + response.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response.getBody shouldBe Validation.empty.withError("name", "name contains whitespace: 'name${*'") + } + "imported dataset has unsupported/incorrect metadata exportVersion" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + propertyDefinitionFixture.add(PropertyDefinitionFactory.getDummyPropertyDefinition("key1")) // key2 propdef is missing + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key2") + ) + + val response1 = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", + bodyOpt = Some(importableDs(metadataContent = """"exportVersion":6"""))) + + response1.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response1.getBody shouldBe Validation.empty.withError("metadata.exportApiVersion", + "Export/Import API version mismatch. Acceptable version is 1. Version passed is 6") + + val response2 = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", + bodyOpt = Some(importableDs(metadataContent = """"otherMetaKey":"otherMetaVal""""))) + + response2.getStatusCode shouldBe HttpStatus.BAD_REQUEST + response2.getBody shouldBe Validation.empty.withError("metadata.exportApiVersion", + "Export/Import API version mismatch. Acceptable version is 1. Version passed is null") + } + "exists and is disabled" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // import feature checks schema presence + val dataset1 = DatasetFactory.getDummyDataset(name = "datasetXYZ", description = Some("init version"), disabled = true) + datasetFixture.add(dataset1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("key1"), + PropertyDefinitionFactory.getDummyPropertyDefinition("key2") + ) + + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs())) + assertBadRequest(response) + + response.getBody shouldBe Validation.empty.withError("disabled", "Entity datasetXYZ is disabled!") + } } "return 201" when { @@ -446,7 +496,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("key3", essentiality = Essentiality.Recommended) ) - val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPost[String, Validation](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs())) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetXYZ/2") @@ -476,7 +526,7 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA PropertyDefinitionFactory.getDummyPropertyDefinition("key2") ) - val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs)) + val response = sendPost[String, String](s"$apiUrl/datasetXYZ/import", bodyOpt = Some(importableDs())) assertCreated(response) val locationHeader = response.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/datasets/datasetXYZ/1") // this is the first version @@ -684,6 +734,22 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA assertBadRequest(response2) response2.getBody shouldBe Validation(Map("AorB" -> List("Value 'c' is not one of the allowed values (a, b)."))) } + "when the dataset is disabled" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", version = 1, disabled = true) + datasetFixture.add(datasetV1) + + propertyDefinitionFixture.add( + PropertyDefinitionFactory.getDummyPropertyDefinition("AorB", propertyType = EnumPropertyType("a", "b")) + ) + + val response = sendPut[Map[String, String], Validation](s"$apiUrl/datasetA/1/properties", + bodyOpt = Some(Map("AorB" -> "a"))) + + assertBadRequest(response) + val responseBody = response.getBody + responseBody shouldBe Validation(Map("disabled" -> List("Entity datasetA is disabled!"))) + } } "201 Created with location" when { @@ -772,6 +838,15 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA assertOk(response) response.getBody shouldBe Validation(Map("AorB" -> List("Value 'c' is not one of the allowed values (a, b)."))) } + "dataset is disabled" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true) + datasetFixture.add(datasetV1) + + val response = sendGet[Validation](s"$apiUrl/datasetA/1/validation") + assertOk(response) + response.getBody shouldBe Validation(Map("disabled" -> List("Entity datasetA is disabled!"))) + } } } @@ -870,6 +945,17 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA response.getBody shouldBe Validation.empty.withError("mapping-table", "Mapping table CurrencyMappingTable v9 not found!") } + "when dataset is disabled" in { + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) + val datasetV1 = DatasetFactory.getDummyDataset(name = "datasetA", disabled = true, conformance = List( + LiteralConformanceRule(order = 0, "column1", true, "ABC")) + ) + datasetFixture.add(datasetV1) + + val response = sendPost[ConformanceRule, Validation](s"$apiUrl/datasetA/1/rules", bodyOpt = Some(exampleLitRule1)) + assertBadRequest(response) + response.getBody shouldBe Validation.empty.withError("disabled", "Entity datasetA is disabled!") + } } "return 201" when { @@ -929,6 +1015,9 @@ class DatasetControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeA } } + // todo add enable cases where dependencies are disabled/removed(?) -> should fail #2065 + // todo add enable cases where dependencies are ok -> should fail #2065 + s"PUT $apiUrl/{name}" can { "return 200" when { "a Dataset with the given name exists" should { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala index 4f975dcc0..b89d599fe 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/MappingTableControllerV3IntegrationSuite.scala @@ -20,13 +20,12 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.http.HttpStatus import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.model.conformanceRule.MappingConformanceRule import za.co.absa.enceladus.model.dataFrameFilter._ import za.co.absa.enceladus.model.menas.MenasReference -import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, PropertyDefinitionFactory, SchemaFactory} +import za.co.absa.enceladus.model.test.factories.{DatasetFactory, MappingTableFactory, SchemaFactory} import za.co.absa.enceladus.model.{DefaultValue, MappingTable, UsedIn, Validation} import za.co.absa.enceladus.rest_api.integration.controllers.{BaseRestApiTestV3, toExpected} import za.co.absa.enceladus.rest_api.integration.fixtures._ @@ -122,7 +121,6 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be )) ) - val response = sendGet[Array[DefaultValue]](s"$apiUrl/mtA/latest/defaults") assertOk(response) response.getBody shouldBe Array(DefaultValue("columnX", "defaultXvalue"), DefaultValue("columnY", "defaultYvalue")) @@ -142,7 +140,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be } "return 400" when { - "when version is not the latest (only last version can be updated)" in { + "the version is not the latest (only last version can be updated)" in { val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1) val mtAv2 = MappingTableFactory.getDummyMappingTable("mtA", version = 2) val mtAv3 = MappingTableFactory.getDummyMappingTable("mtA", version = 3) @@ -157,6 +155,18 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be List("Version 2 of mtA is not the latest version, therefore cannot be edited") )) } + "the mapping table is disabled" in { + val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1, disabled = true) + .copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) + mappingTableFixture.add(mtAv1) + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist + + val response = sendPut[Array[DefaultValue], Validation](s"$apiUrl/mtA/1/defaults", + bodyOpt = Some(Array.empty[DefaultValue])) + assertBadRequest(response) + + response.getBody shouldBe Validation.empty.withError("disabled", "Entity mtA is disabled!") + } } "201 Created with location" when { @@ -213,10 +223,20 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be List("Version 2 of mtA is not the latest version, therefore cannot be edited") )) } + "the mapping table is disabled" in { + val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1, disabled = true) + .copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) + mappingTableFixture.add(mtAv1) + schemaFixture.add(SchemaFactory.getDummySchema("dummySchema")) // Schema referenced by MT must exist + + val response = sendPost[DefaultValue, Validation](s"$apiUrl/mtA/1/defaults", bodyOpt = Some(DefaultValue("colA", "defaultA"))) + assertBadRequest(response) + response.getBody shouldBe Validation.empty.withError("disabled", "Entity mtA is disabled!") + } } "201 Created with location" when { - s"defaults are replaced with a new version" in { + "defaults are replaced with a new version" in { val mtAv1 = MappingTableFactory.getDummyMappingTable("mtA", version = 1).copy(defaultMappingValue = List(DefaultValue("anOldDefault", "itsValue"))) mappingTableFixture.add(mtAv1) @@ -265,9 +285,9 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) mappingTableFixture.add(mappingTable1, mappingTable2) - val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List(mcr("mappingTable",1))) - val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", conformance = List(mcr("mappingTable",1)), disabled = true) - val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable",2))) + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List(mcr("mappingTable", 1))) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", conformance = List(mcr("mappingTable", 1)), disabled = true) + val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable", 2))) datasetFixture.add(datasetA, datasetB, datasetC) val response = sendGet[String](s"$apiUrl/mappingTable/used-in") @@ -294,9 +314,9 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val mappingTable2 = MappingTableFactory.getDummyMappingTable(name = "mappingTable", version = 2) mappingTableFixture.add(mappingTable1, mappingTable2) - val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List(mcr("mappingTable",1))) - val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", conformance = List(mcr("mappingTable",1)), disabled = true) - val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable",2))) + val datasetA = DatasetFactory.getDummyDataset(name = "datasetA", conformance = List(mcr("mappingTable", 1))) + val datasetB = DatasetFactory.getDummyDataset(name = "datasetB", conformance = List(mcr("mappingTable", 1)), disabled = true) + val datasetC = DatasetFactory.getDummyDataset(name = "datasetC", conformance = List(mcr("mappingTable", 2))) datasetFixture.add(datasetA, datasetB, datasetC) val response = sendGet[UsedIn](s"$apiUrl/mappingTable/1/used-in") @@ -408,7 +428,7 @@ class MappingTableControllerV3IntegrationSuite extends BaseRestApiTestV3 with Be val dataset1 = DatasetFactory.getDummyDataset(name = "dataset1", conformance = List(mcr("mappingTable", 1))) val dataset2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 7, conformance = List(mcr("mappingTable", 2))) - val dataset3 = DatasetFactory.getDummyDataset(name = "dataset3",conformance = List(mcr("anotherMappingTable", 8))) // moot + val dataset3 = DatasetFactory.getDummyDataset(name = "dataset3", conformance = List(mcr("anotherMappingTable", 8))) // moot val disabledDs = DatasetFactory.getDummyDataset(name = "disabledDs", conformance = List(mcr("mappingTable", 2)), disabled = true) datasetFixture.add(dataset1, dataset2, dataset3, disabledDs) diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala index 2c325405c..aa1c0a233 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/PropertyDefinitionControllerV3IntegrationSuite.scala @@ -240,6 +240,22 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w response2.getBody should include("name mismatch: 'propertyDefinitionABC' != 'propertyDefinitionXYZ'") } } + "the pd is disabled" in { + val propertyDefinitionA1 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA") + val propertyDefinitionA2 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA", + description = Some("second version"), version = 2, disabled = true) + propertyDefinitionFixture.add(propertyDefinitionA1, propertyDefinitionA2) + + val propertyDefinitionA3 = PropertyDefinitionFactory.getDummyPropertyDefinition("propertyDefinitionA", + description = Some("updated"), + propertyType = EnumPropertyType("a", "b"), + version = 2 // update references the last version + ) + + val response = sendPutByAdmin[PropertyDefinition, Validation](s"$apiUrl/propertyDefinitionA/2", bodyOpt = Some(propertyDefinitionA3)) + assertBadRequest(response) + response.getBody shouldBe Validation.empty.withError("disabled", "Entity propertyDefinitionA is disabled!") + } } "return 403" when { @@ -271,7 +287,7 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w s"POST $apiUrl/{name}/import" should { val importablePd = - """{"todo":{"exportVersion":1},"item":{ + """{"metadata":{"exportVersion":1},"item":{ |"name":"propertyDefinitionXYZ", |"description":"Hi, I am the import", |"propertyType":{"_t":"StringPropertyType"}, @@ -288,6 +304,16 @@ class PropertyDefinitionControllerV3IntegrationSuite extends BaseRestApiTestV3 w response.getBody should include("name mismatch: 'propertyDefinitionABC' != 'propertyDefinitionXYZ'") } } + "the pd is disabled" in { + val propertyDefinition1 = PropertyDefinitionFactory.getDummyPropertyDefinition(name = "propertyDefinitionXYZ", + description = Some("init version"), disabled = true) + propertyDefinitionFixture.add(propertyDefinition1) + + val response = sendPostByAdmin[String, Validation](s"$apiUrl/propertyDefinitionXYZ/import", bodyOpt = Some(importablePd)) + assertBadRequest(response) + response.getBody shouldBe Validation.empty.withError("disabled", "Entity propertyDefinitionXYZ is disabled!") + + } } "return 403" when { diff --git a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala index d03194ac5..1b4898402 100644 --- a/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala +++ b/rest-api/src/test/scala/za/co/absa/enceladus/rest_api/integration/controllers/v3/SchemaControllerV3IntegrationSuite.scala @@ -169,6 +169,20 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn response.getBody shouldBe Validation.empty .withError("schema-fields", "No fields found! There must be fields defined for actual usage.") } + "schema is disabled" in { + val schema1 = SchemaFactory.getDummySchema("schemaA", disabled = true, fields = List( + SchemaField("field1", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty) + )) + schemaFixture.add(schema1) + + val schema2 = SchemaFactory.getDummySchema("schemaA", fields = List( + SchemaField("anotherField", "string", "", nullable = true, metadata = Map.empty, children = Seq.empty) + )) + val response = sendPut[Schema, Validation](s"$apiUrl/schemaA/1", bodyOpt = Some(schema2)) + + assertBadRequest(response) + response.getBody shouldBe Validation.empty.withError("disabled", "Entity schemaA is disabled!") + } } } @@ -502,6 +516,16 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } } + "schema is disabled" in { + val schema = SchemaFactory.getDummySchema("schemaA", disabled = true) + schemaFixture.add(schema) + + val schemaParams = HashMap[String, Any]("format" -> "struct") + val responseUploaded = sendPostUploadFile[Validation]( + s"$apiUrl/schemaA/1/from-file", TestResourcePath.Json.ok, schemaParams) + assertBadRequest(responseUploaded) + responseUploaded.getBody shouldBe Validation.empty.withError("disabled", "Entity schemaA is disabled!") + } } "return 404" when { @@ -546,8 +570,10 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Copybook.ok))) val params = HashMap[String, Any]("format" -> "copybook", "remoteUrl" -> remoteUrl) - val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFile[Validation](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) + responseRemoteLoaded.getBody shouldBe Validation.empty + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -570,8 +596,9 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Json.ok))) val params = HashMap("remoteUrl" -> remoteUrl, "format" -> "struct") - val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFile[Validation](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) + responseRemoteLoaded.getBody shouldBe Validation.empty val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -594,8 +621,10 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "remoteUrl" -> remoteUrl) - val responseRemoteLoaded = sendPostRemoteFile[Schema](s"$apiUrl/schemaA/1/from-remote-uri", params) + val responseRemoteLoaded = sendPostRemoteFile[Validation](s"$apiUrl/schemaA/1/from-remote-uri", params) assertCreated(responseRemoteLoaded) + responseRemoteLoaded.getBody shouldBe Validation.empty + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -657,6 +686,19 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } } + "the schema is disabled" in { + val schema = SchemaFactory.getDummySchema("schemaA", disabled = true) + schemaFixture.add(schema) + + wireMockServer.stubFor(get(urlPathEqualTo(remoteFilePath)) + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) + + val params = HashMap[String, Any]("format" -> "avro", "remoteUrl" -> remoteUrl) + val responseRemoteLoaded = sendPostRemoteFile[Validation](s"$apiUrl/schemaA/1/from-remote-uri", params) + assertBadRequest(responseRemoteLoaded) + responseRemoteLoaded.getBody shouldBe Validation.empty.withError("disabled", "Entity schemaA is disabled!") + + } } "return 404" when { @@ -684,8 +726,10 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic1-value") - val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/schemaA/1/from-registry", params) + val responseRemoteLoaded = sendPostSubject[Validation](s"$apiUrl/schemaA/1/from-registry", params) assertCreated(responseRemoteLoaded) + responseRemoteLoaded.getBody shouldBe Validation.empty + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -709,8 +753,10 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic2") - val responseRemoteLoaded = sendPostSubject[Schema](s"$apiUrl/schemaA/1/from-registry", params) + val responseRemoteLoaded = sendPostSubject[Validation](s"$apiUrl/schemaA/1/from-registry", params) assertCreated(responseRemoteLoaded) + responseRemoteLoaded.getBody shouldBe Validation.empty + val locationHeader = responseRemoteLoaded.getHeaders.getFirst("location") locationHeader should endWith("/api-v3/schemas/schemaA/2") // +1 version @@ -724,6 +770,21 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } } + + "return 400" when { + "the schema is disabled" in { + val schema = SchemaFactory.getDummySchema("schemaA", disabled = true) + schemaFixture.add(schema) + + wireMockServer.stubFor(get(urlPathEqualTo(subjectPath("myTopic1-value"))) + .willReturn(readTestResourceAsResponseWithContentType(TestResourcePath.Avro.ok))) + + val params = HashMap[String, Any]("format" -> "avro", "subject" -> "myTopic1-value") + val responseRemoteLoaded = sendPostSubject[Validation](s"$apiUrl/schemaA/1/from-registry", params) + assertBadRequest(responseRemoteLoaded) + responseRemoteLoaded.getBody shouldBe Validation.empty.withError("disabled", "Entity schemaA is disabled!") + } + } } s"GET $apiUrl/{name}/used-in" should { @@ -1015,5 +1076,4 @@ class SchemaControllerV3IntegrationSuite extends BaseRestApiTestV3 with BeforeAn } } } - }