From c96f779ade64715f26309def34eee40c99507b22 Mon Sep 17 00:00:00 2001 From: Adrian Olosutean Date: Tue, 31 Aug 2021 12:52:45 +0300 Subject: [PATCH] 1868 statistics with missing counts and datasets missing proprties (#1873) * 1868 statistics with missing counts and datasets missing proprties --- .../properties/PropertyDefinitionStats.scala | 30 +++++ .../menas/controllers/DatasetController.scala | 10 ++ .../controllers/LandingPageController.scala | 40 +++++-- .../controllers/StatisticsController.scala | 36 ++++++ .../menas/models/LandingPageInformation.scala | 3 + .../VersionedMongoRepository.scala | 25 ++-- .../menas/services/DatasetService.scala | 18 +-- .../services/PropertyDefinitionService.scala | 7 +- .../menas/services/StatisticsService.scala | 45 +++++++ .../services/VersionedModelService.scala | 2 +- ...ropertyDefinitionApiIntegrationSuite.scala | 2 +- .../DatasetRepositoryIntegrationSuite.scala | 24 +++- .../StatisticsIntegrationSuite.scala | 110 ++++++++++++++++++ 13 files changed, 318 insertions(+), 34 deletions(-) create mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala create mode 100644 menas/src/main/scala/za/co/absa/enceladus/menas/controllers/StatisticsController.scala create mode 100644 menas/src/main/scala/za/co/absa/enceladus/menas/services/StatisticsService.scala create mode 100644 menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/StatisticsIntegrationSuite.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala new file mode 100644 index 000000000..fa933a5de --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model.properties + +import za.co.absa.enceladus.model.properties.essentiality.Essentiality + +case class PropertyDefinitionStats(name: String, + version: Int = 1, + essentiality: Essentiality = Essentiality.Optional, + missingInDatasetsCount: Int = 0) + +object PropertyDefinitionStats { + def apply(propertyDefinition: PropertyDefinition, missingCounts: Int): PropertyDefinitionStats = { + PropertyDefinitionStats(propertyDefinition.name, propertyDefinition.version, + propertyDefinition.essentiality, missingCounts) + } +} diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/DatasetController.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/DatasetController.scala index 953a9105b..9d4ad441e 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/DatasetController.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/DatasetController.scala @@ -19,6 +19,7 @@ import java.net.URI import java.util import java.util.Optional import java.util.concurrent.CompletableFuture + import org.slf4j.{Logger, LoggerFactory} import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.{HttpStatus, ResponseEntity} @@ -29,6 +30,7 @@ import za.co.absa.enceladus.menas.services.DatasetService import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import za.co.absa.enceladus.model.conformanceRule.ConformanceRule import za.co.absa.enceladus.model.properties.PropertyDefinition +import za.co.absa.enceladus.model.versionedModel.VersionedSummary import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel.Constants.DefaultValidationLevelName @@ -44,6 +46,14 @@ class DatasetController @Autowired()(datasetService: DatasetService) import scala.concurrent.ExecutionContext.Implicits.global + @GetMapping(Array("/latest")) + @ResponseStatus(HttpStatus.OK) + def getLatestVersions(@RequestParam(value = "missing_property", required = false) + missingProperty: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + datasetService.getLatestVersions(missingProperty.toScalaOption) + .map(datasets => datasets.map(dataset => VersionedSummary(dataset.name, dataset.version))) + } + @PostMapping(Array("/{datasetName}/rule/create")) @ResponseStatus(HttpStatus.OK) def addConformanceRule(@AuthenticationPrincipal user: UserDetails, diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala index e372b5be8..b7832d706 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala @@ -18,20 +18,19 @@ package za.co.absa.enceladus.menas.controllers import java.util.concurrent.CompletableFuture import scala.concurrent.Future - import org.springframework.beans.factory.annotation.Autowired import org.springframework.scheduling.annotation.Async import org.springframework.scheduling.annotation.Scheduled import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RestController - import za.co.absa.enceladus.menas.models.LandingPageInformation import za.co.absa.enceladus.menas.repositories.DatasetMongoRepository import za.co.absa.enceladus.menas.repositories.LandingPageStatisticsMongoRepository import za.co.absa.enceladus.menas.repositories.MappingTableMongoRepository import za.co.absa.enceladus.menas.repositories.SchemaMongoRepository -import za.co.absa.enceladus.menas.services.RunService +import za.co.absa.enceladus.menas.services.{PropertyDefinitionService, RunService, StatisticsService} +import za.co.absa.enceladus.model.properties.essentiality.{Mandatory, Recommended} @RestController @RequestMapping(Array("/api/landing")) @@ -39,7 +38,8 @@ class LandingPageController @Autowired() (datasetRepository: DatasetMongoReposit mappingTableRepository: MappingTableMongoRepository, schemaRepository: SchemaMongoRepository, runsService: RunService, - landingPageRepository: LandingPageStatisticsMongoRepository) extends BaseController { + landingPageRepository: LandingPageStatisticsMongoRepository, + statisticsService: StatisticsService) extends BaseController { import scala.concurrent.ExecutionContext.Implicits.global import za.co.absa.enceladus.menas.utils.implicits._ @@ -50,13 +50,31 @@ class LandingPageController @Autowired() (datasetRepository: DatasetMongoReposit } def landingPageInfo(): Future[LandingPageInformation] = { + val dsCountFuture = datasetRepository.distinctCount() + val mappingTableFuture = mappingTableRepository.distinctCount() + val schemaFuture = schemaRepository.distinctCount() + val runFuture = runsService.getCount() + val propertiesWithMissingCountsFuture = statisticsService.getPropertiesWithMissingCount() + val propertiesTotalsFuture: Future[(Int, Int, Int)] = propertiesWithMissingCountsFuture.map(props => { + props.foldLeft(0, 0, 0) { (acum, item) => + val (count, mandatoryCount, recommendedCount) = acum + item.essentiality match { + case Mandatory(_) => (count + 1, mandatoryCount + item.missingInDatasetsCount, recommendedCount) + case Recommended() => (count + 1, mandatoryCount, recommendedCount + item.missingInDatasetsCount) + case _ => (count + 1, mandatoryCount, recommendedCount) + } + } + }) + val todaysStatsfuture = runsService.getTodaysRunsStatistics() for { - dsCount <- datasetRepository.distinctCount() - mtCount <- mappingTableRepository.distinctCount() - schemaCount <- schemaRepository.distinctCount() - runCount <- runsService.getCount() - todaysStats <- runsService.getTodaysRunsStatistics() - } yield LandingPageInformation(dsCount, mtCount, schemaCount, runCount, todaysStats) + dsCount <- dsCountFuture + mtCount <- mappingTableFuture + schemaCount <- schemaFuture + runCount <- runFuture + (propertiesCount, totalMissingMandatoryProperties, totalMissingRecommendedProperties) <- propertiesTotalsFuture + todaysStats <- todaysStatsfuture + } yield LandingPageInformation(dsCount, mtCount, schemaCount, runCount, propertiesCount, + totalMissingMandatoryProperties, totalMissingRecommendedProperties, todaysStats) } // scalastyle:off magic.number @@ -67,6 +85,6 @@ class LandingPageController @Autowired() (datasetRepository: DatasetMongoReposit for { newStats <- landingPageInfo() res <- landingPageRepository.updateStatistics(newStats) - } yield res + } yield res } } diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/StatisticsController.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/StatisticsController.scala new file mode 100644 index 000000000..492d5d303 --- /dev/null +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/StatisticsController.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.menas.controllers + +import java.util.concurrent.CompletableFuture + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.bind.annotation.{GetMapping, RequestMapping, RestController} +import za.co.absa.enceladus.menas.services.StatisticsService +import za.co.absa.enceladus.model.properties.PropertyDefinitionStats + +@RestController +@RequestMapping(Array("/api/statistics")) +class StatisticsController @Autowired() (statisticsService: StatisticsService) extends BaseController { + + import za.co.absa.enceladus.menas.utils.implicits._ + + @GetMapping(Array("/properties/missing")) + def getPropertiesWithMissingCount(): CompletableFuture[Seq[PropertyDefinitionStats]] = { + statisticsService.getPropertiesWithMissingCount() + } + +} diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/models/LandingPageInformation.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/models/LandingPageInformation.scala index 5950e1ded..c4c830e4b 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/models/LandingPageInformation.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/models/LandingPageInformation.scala @@ -20,4 +20,7 @@ case class LandingPageInformation( totalNumberMappingTables: Int, totalNumberSchemas: Int, totalNumberRuns: Long, + totalNumberProperties: Int, + totalNumberMissingMandatoryProperties: Int, + totalNumberMissingRecommendedProperties: Int, todaysRunsStatistics: TodaysRunsStatistics) diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/repositories/VersionedMongoRepository.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/repositories/VersionedMongoRepository.scala index 3fbce015a..bbf4822b8 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/repositories/VersionedMongoRepository.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/repositories/VersionedMongoRepository.scala @@ -75,15 +75,10 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.aggregate[VersionedSummary](pipeline).toFuture() } - def getLatestVersions(): Future[Seq[C]] = { - // there may be a way to this using mongo-joining (aggregation.lookup) instead - getLatestVersionsSummary(None).flatMap { summaries => - val resultIn = summaries.map { summary => - getVersion(summary._id, summary.latestVersion).map(_.toSeq) - } - - Future.sequence(resultIn).map(_.flatten) - } + def getLatestVersions(missingProperty: Option[String]): Future[Seq[C]] = { + val missingFilter = missingProperty.map(missingProp => + Filters.not(Filters.exists(s"properties.$missingProp"))) + collectLatestVersions(missingFilter) } def getVersion(name: String, version: Int): Future[Option[C]] = { @@ -163,6 +158,18 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab .toFuture() } + private def collectLatestVersions(postAggFilter: Option[Bson]): Future[Seq[C]] = { + val pipeline = Seq( + filter(Filters.notEqual("disabled", true)), + Aggregates.group("$name", + Accumulators.max("latestVersion", "$version"), + Accumulators.last("doc","$$ROOT")), + Aggregates.replaceRoot("$doc")) ++ + postAggFilter.map(Aggregates.filter) + + collection.aggregate[C](pipeline).toFuture() + } + private[repositories] def getNotDisabledFilter: Bson = { notEqual("disabled", true) } diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala index 40527fea1..b7d9a8835 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala @@ -15,24 +15,21 @@ package za.co.absa.enceladus.menas.services -import scala.concurrent.Future import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.menas.repositories.DatasetMongoRepository -import za.co.absa.enceladus.menas.repositories.OozieRepository -import za.co.absa.enceladus.model.{Dataset, Schema, UsedIn, Validation} +import za.co.absa.enceladus.menas.repositories.{DatasetMongoRepository, OozieRepository} +import za.co.absa.enceladus.menas.services.DatasetService.{RuleValidationsAndFields, _} import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, _} import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieScheduleInstance - -import scala.language.reflectiveCalls -import DatasetService.RuleValidationsAndFields -import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ import za.co.absa.enceladus.model.properties.essentiality.Mandatory +import za.co.absa.enceladus.model.{Dataset, Schema, UsedIn, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel -import DatasetService._ +import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel +import scala.concurrent.Future +import scala.language.reflectiveCalls import scala.util.{Failure, Success} @@ -217,6 +214,9 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } + def getLatestVersions(missingProperty: Option[String]): Future[Seq[Dataset]] = + datasetMongoRepository.getLatestVersions(missingProperty) + override def importItem(item: Dataset, username: String): Future[Option[Dataset]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/PropertyDefinitionService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/PropertyDefinitionService.scala index 2041c3f45..a7851d542 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/services/PropertyDefinitionService.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/PropertyDefinitionService.scala @@ -17,8 +17,7 @@ package za.co.absa.enceladus.menas.services import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.menas.repositories.{DatasetMongoRepository, PropertyDefinitionMongoRepository} -import za.co.absa.enceladus.menas.utils.converters.SparkMenasSchemaConvertor +import za.co.absa.enceladus.menas.repositories.PropertyDefinitionMongoRepository import za.co.absa.enceladus.model.UsedIn import za.co.absa.enceladus.model.properties.PropertyDefinition @@ -42,6 +41,10 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert } } + def getDistinctCount(): Future[Int] = { + propertyDefMongoRepository.distinctCount() + } + override def create(newPropertyDef: PropertyDefinition, username: String): Future[Option[PropertyDefinition]] = { val propertyDefBase = PropertyDefinition( name = newPropertyDef.name, diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/StatisticsService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/StatisticsService.scala new file mode 100644 index 000000000..9f55f4d21 --- /dev/null +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/StatisticsService.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.menas.services + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component +import za.co.absa.enceladus.model.properties.{PropertyDefinition, PropertyDefinitionStats} + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +@Component +class StatisticsService @Autowired() (propertyDefService: PropertyDefinitionService, datasetService: DatasetService){ + //#TODO find optimizations #1897 + def getPropertiesWithMissingCount(): Future[Seq[PropertyDefinitionStats]] = { + val propertyDefsFuture = propertyDefService.getLatestVersions() + propertyDefsFuture + .map { (props: Seq[PropertyDefinition]) => + val propertiesWithMissingCounts: Seq[Future[PropertyDefinitionStats]] = props.map(propertyDef => + datasetService + .getLatestVersions(Some(propertyDef.name)) + .map(datasetsMissingProp => + PropertyDefinitionStats(propertyDef, datasetsMissingProp.size)) + ) + propertiesWithMissingCounts + } + .flatMap { propertiesWithMissingCounts: Seq[Future[PropertyDefinitionStats]] => + Future.sequence(propertiesWithMissingCounts) + } + } + +} diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/VersionedModelService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/VersionedModelService.scala index a486492d5..7f7b4954c 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/services/VersionedModelService.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/VersionedModelService.scala @@ -41,7 +41,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit } def getLatestVersions(): Future[Seq[C]] = { - versionedMongoRepository.getLatestVersions() + versionedMongoRepository.getLatestVersions(None) } def getSearchSuggestions(): Future[Seq[String]] = { diff --git a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala index fbd5a8346..e81d14694 100644 --- a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala +++ b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala @@ -384,7 +384,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTest with BeforeA val response = sendGet[Array[PropertyDefinition]](s"$apiUrl") // Array to avoid erasure assertOk(response) - val responseData = response.getBody.toSeq.map(pd => (pd.name, pd.version)) + val responseData = response.getBody.toSeq.map(pd => (pd.name, pd.version)).sortBy(_._1) val expectedData = Seq("propertyDefinitionA" -> 2, "propertyDefinitionB" -> 3) // disabled pdA-v3 not reported assert(responseData == expectedData) } diff --git a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/DatasetRepositoryIntegrationSuite.scala index eb8838781..2f5be1aee 100644 --- a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -24,6 +24,7 @@ import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.menas.exceptions.EntityAlreadyExistsException import za.co.absa.enceladus.menas.integration.fixtures.{DatasetFixtureService, FixtureService} import za.co.absa.enceladus.menas.repositories.DatasetMongoRepository +import za.co.absa.enceladus.model.Dataset import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.test.factories.DatasetFactory import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieSchedule @@ -512,6 +513,27 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val expected = Seq(dataset3, dataset4).map(DatasetFactory.toSummary) assert(actual == expected) } + + "search with missing properties" in { + val dataset1ver1 = DatasetFactory.getDummyDataset(name = "dataset1", version = 1) + val dataset1ver2 = DatasetFactory.getDummyDataset(name = "dataset1", version = 2, + properties = Some(Map("prop1"->"a"))) + val dataset2ver1 = DatasetFactory.getDummyDataset(name = "dataset2", version = 1) + val dataset2ver2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 2) + val dataset3ver1 = DatasetFactory.getDummyDataset(name = "dataset3", version = 1) + val dataset4ver1 = DatasetFactory.getDummyDataset(name = "dataset4", version = 1, + properties = Some(Map("prop1"->"A"))) + + val abc1 = DatasetFactory.getDummyDataset(name = "abc", version = 1) + + datasetFixture.add(dataset1ver1, dataset1ver2, dataset2ver1, dataset2ver2, dataset3ver1, dataset4ver1, abc1) + + val actual: Seq[Dataset] = await(datasetMongoRepository.getLatestVersions(Some("prop1"))) + .sortBy(_.name) + + val expected = Seq(abc1, dataset2ver2, dataset3ver1) + assert(actual == expected) + } } "return all datasets" when { @@ -595,7 +617,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { assert(await(datasetMongoRepository.findByCoordId("SomeCoordId")) == Seq()) } } - "return datasets witch matching coordinator ID" when { + "return datasets with matching coordinator ID" when { "such datasets exist" in { val schedule = OozieSchedule(scheduleTiming = ScheduleTiming(Seq(), Seq(), Seq(), Seq(), Seq()), runtimeParams = RuntimeConfig(sysUser = "user", menasKeytabFile = "/a/b/c"), datasetVersion = 0, diff --git a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/StatisticsIntegrationSuite.scala b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/StatisticsIntegrationSuite.scala new file mode 100644 index 000000000..69356fc11 --- /dev/null +++ b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/StatisticsIntegrationSuite.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.menas.integration.repositories + +import org.junit.runner.RunWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.junit4.SpringRunner +import za.co.absa.enceladus.menas.integration.fixtures.{DatasetFixtureService, FixtureService, PropertyDefinitionFixtureService} +import za.co.absa.enceladus.menas.repositories.{DatasetMongoRepository, PropertyDefinitionMongoRepository} +import za.co.absa.enceladus.menas.services.StatisticsService +import za.co.absa.enceladus.model.properties.{PropertyDefinition, PropertyDefinitionStats} +import za.co.absa.enceladus.model.properties.essentiality.Essentiality.{Mandatory, Optional, Recommended} +import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, StringPropertyType} +import za.co.absa.enceladus.model.test.factories.DatasetFactory + +@RunWith(classOf[SpringRunner]) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(Array("withEmbeddedMongo")) +class StatisticsIntegrationSuite extends BaseRepositoryTest { + @Autowired + private val datasetFixture: DatasetFixtureService = null + + @Autowired + private val propertyDefService: PropertyDefinitionFixtureService = null + + @Autowired + private val statisticsService: StatisticsService = null + + override def fixtures: List[FixtureService[_]] = List(datasetFixture) + + val mockedPropertyDefinitions = Seq( + PropertyDefinition(name = "mandatoryString1", propertyType = StringPropertyType(), essentiality = Mandatory(allowRun = false), + userCreated = "", userUpdated = ""), + PropertyDefinition(name = "mandatoryString2", propertyType = StringPropertyType(), essentiality = Mandatory(allowRun = false), + userCreated = "", userUpdated = ""), + PropertyDefinition(name = "recommendedString1", propertyType = StringPropertyType(), essentiality = Recommended, + userCreated = "", userUpdated = ""), + PropertyDefinition(name = "optionalString1", propertyType = StringPropertyType(), essentiality = Optional, + userCreated = "", userUpdated = ""), + PropertyDefinition(name = "mandatoryDisabledString1", propertyType = StringPropertyType(), essentiality = Mandatory(allowRun = false), + disabled = true, userCreated = "", userUpdated = ""), + PropertyDefinition(name = "optionalEnumAb", propertyType = EnumPropertyType("optionA", "optionB"), essentiality = Optional, + userCreated = "", userUpdated = "") + ) + + val mockedDatasets = Seq( + DatasetFactory.getDummyDataset(name = "dataset1", version = 1, properties = Some( + Map())), + DatasetFactory.getDummyDataset(name = "dataset1", version = 2, properties = Some( + Map("mandatoryString1"->""))), + DatasetFactory.getDummyDataset(name = "dataset1", version = 3, properties = Some( + Map("mandatoryString1"->"", "mandatoryString2"->"3", "optionalEnumAb" -> "optionA"))), + DatasetFactory.getDummyDataset(name = "dataset2", version = 1, properties = Some( + Map("recommendedString1" -> "", "optionalString1"->""))), + DatasetFactory.getDummyDataset(name = "dataset2", version = 2, properties = Some( + Map("mandatoryString1"->"", "recommendedString1" -> ""))), + DatasetFactory.getDummyDataset(name = "dataset3", version = 1, properties = Some( + Map("mandatoryString1"->""))), + DatasetFactory.getDummyDataset(name = "dataset3", version = 2, properties = Some( + Map("mandatoryString1"->"","mandatoryString2"->"3"))), + DatasetFactory.getDummyDataset(name = "dataset4", version = 1, properties = Some( + Map("mandatoryString1"->"", "mandatoryString2"->"3", "recommendedString1" -> "", "optionalString1"->"", + "mandatoryDisabledString1" -> "", "optionalEnumAb" -> "optionA"))), + DatasetFactory.getDummyDataset(name = "dataset5", version = 1, properties = Some(Map())), + DatasetFactory.getDummyDataset(name = "dataset6", version = 1, properties = Some(Map( + "mandatoryString1"->"", "mandatoryString2"->"3", "recommendedString1" -> ""))) + ) + + "StatisticsService" should { + + "return the properties with missing counts" when { + "the specified datasets and properties" in { + datasetFixture.add(mockedDatasets: _*) + propertyDefService.add(mockedPropertyDefinitions: _*) + + val actualStatistics = await(statisticsService.getPropertiesWithMissingCount()).sortBy(_.name) + + val expectedStatistics = Seq( + PropertyDefinitionStats(name = "mandatoryString1", essentiality = Mandatory(allowRun = false), + missingInDatasetsCount = 1), // missing in dataset5 + PropertyDefinitionStats(name = "mandatoryString2", essentiality = Mandatory(allowRun = false), + missingInDatasetsCount = 2), // missing in dataset2,5 + PropertyDefinitionStats(name = "optionalEnumAb", essentiality = Optional, + missingInDatasetsCount = 4), // missing in dataset2,3,5,6 + PropertyDefinitionStats(name = "optionalString1", essentiality = Optional, + missingInDatasetsCount = 5), // missing in dataset1,2,3,5,6 + PropertyDefinitionStats(name = "recommendedString1", essentiality = Recommended, + missingInDatasetsCount = 3) // missing in dataset1,3,5 + ) + + assert(actualStatistics == expectedStatistics) + } + } + } +}