From bc787c44c1ba6237e185490c55cf9ad1fc0e60f9 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sat, 21 Aug 2021 10:20:12 +0200 Subject: [PATCH 01/21] Update for next development version 2.24.0-SNAPSHOT --- dao/pom.xml | 2 +- data-model/pom.xml | 2 +- examples/pom.xml | 2 +- menas/pom.xml | 2 +- migrations-cli/pom.xml | 2 +- migrations/pom.xml | 2 +- plugins-api/pom.xml | 2 +- plugins-builtin/pom.xml | 2 +- pom.xml | 2 +- spark-jobs/pom.xml | 2 +- utils/pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dao/pom.xml b/dao/pom.xml index 3cb416326..987d585b0 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/data-model/pom.xml b/data-model/pom.xml index 801d69b6f..ac6c37679 100644 --- a/data-model/pom.xml +++ b/data-model/pom.xml @@ -24,7 +24,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 23d93ba72..3d0d465ec 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/menas/pom.xml b/menas/pom.xml index 1be9866a1..20c6c2aa2 100644 --- a/menas/pom.xml +++ b/menas/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/migrations-cli/pom.xml b/migrations-cli/pom.xml index eae337e6f..70ada3329 100644 --- a/migrations-cli/pom.xml +++ b/migrations-cli/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/migrations/pom.xml b/migrations/pom.xml index 335051273..09aafc7de 100644 --- a/migrations/pom.xml +++ b/migrations/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/plugins-api/pom.xml b/plugins-api/pom.xml index 5f6c505f8..71f0b8355 100644 --- a/plugins-api/pom.xml +++ b/plugins-api/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/plugins-builtin/pom.xml b/plugins-builtin/pom.xml index 714e83933..79a2d9fe1 100644 --- a/plugins-builtin/pom.xml +++ b/plugins-builtin/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 3c9f499e7..11ceff80a 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT pom Enceladus diff --git a/spark-jobs/pom.xml b/spark-jobs/pom.xml index 75b911139..8add8a18e 100644 --- a/spark-jobs/pom.xml +++ b/spark-jobs/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT diff --git a/utils/pom.xml b/utils/pom.xml index 3bf6b47ea..d50c54070 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.23.0 + 2.24.0-SNAPSHOT From 6e8055af90b888040fb6a7d9f978651bb8004911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20Zejnilovi=C4=87?= Date: Mon, 23 Aug 2021 12:13:02 +0200 Subject: [PATCH 02/21] Suppress download noise in license check --- .github/workflows/license_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index f8c7697bb..916047963 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -28,4 +28,4 @@ jobs: - uses: actions/setup-java@v1 with: java-version: 1.8 - - run: mvn -Plicense-check apache-rat:check + - run: mvn --no-transfer-progress -Plicense-check apache-rat:check From a4998ad99a84130a369016a15146874c4f12911b Mon Sep 17 00:00:00 2001 From: David Benedeki <14905969+benedeki@users.noreply.github.com> Date: Tue, 24 Aug 2021 20:06:21 +0200 Subject: [PATCH 03/21] Suppress compiler warning of obsolete Java (#1892) --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 11ceff80a..7a3112886 100644 --- a/pom.xml +++ b/pom.xml @@ -122,6 +122,8 @@ UTF-8 UTF-8 + 1.8 + 1.8 1.4.13 1.16.0 From c96f779ade64715f26309def34eee40c99507b22 Mon Sep 17 00:00:00 2001 From: Adrian Olosutean Date: Tue, 31 Aug 2021 12:52:45 +0300 Subject: [PATCH 04/21] 1868 statistics with missing counts and datasets missing proprties (#1873) * 1868 statistics with missing counts and datasets missing proprties --- .../properties/PropertyDefinitionStats.scala | 30 +++++ .../menas/controllers/DatasetController.scala | 10 ++ .../controllers/LandingPageController.scala | 40 +++++-- .../controllers/StatisticsController.scala | 36 ++++++ .../menas/models/LandingPageInformation.scala | 3 + .../VersionedMongoRepository.scala | 25 ++-- .../menas/services/DatasetService.scala | 18 +-- .../services/PropertyDefinitionService.scala | 7 +- .../menas/services/StatisticsService.scala | 45 +++++++ .../services/VersionedModelService.scala | 2 +- ...ropertyDefinitionApiIntegrationSuite.scala | 2 +- .../DatasetRepositoryIntegrationSuite.scala | 24 +++- .../StatisticsIntegrationSuite.scala | 110 ++++++++++++++++++ 13 files changed, 318 insertions(+), 34 deletions(-) create mode 100644 data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala create mode 100644 menas/src/main/scala/za/co/absa/enceladus/menas/controllers/StatisticsController.scala create mode 100644 menas/src/main/scala/za/co/absa/enceladus/menas/services/StatisticsService.scala create mode 100644 menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/StatisticsIntegrationSuite.scala diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala new file mode 100644 index 000000000..fa933a5de --- /dev/null +++ b/data-model/src/main/scala/za/co/absa/enceladus/model/properties/PropertyDefinitionStats.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.model.properties + +import za.co.absa.enceladus.model.properties.essentiality.Essentiality + +case class PropertyDefinitionStats(name: String, + version: Int = 1, + essentiality: Essentiality = Essentiality.Optional, + missingInDatasetsCount: Int = 0) + +object PropertyDefinitionStats { + def apply(propertyDefinition: PropertyDefinition, missingCounts: Int): PropertyDefinitionStats = { + PropertyDefinitionStats(propertyDefinition.name, propertyDefinition.version, + propertyDefinition.essentiality, missingCounts) + } +} diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/DatasetController.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/DatasetController.scala index 953a9105b..9d4ad441e 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/DatasetController.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/DatasetController.scala @@ -19,6 +19,7 @@ import java.net.URI import java.util import java.util.Optional import java.util.concurrent.CompletableFuture + import org.slf4j.{Logger, LoggerFactory} import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.{HttpStatus, ResponseEntity} @@ -29,6 +30,7 @@ import za.co.absa.enceladus.menas.services.DatasetService import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import za.co.absa.enceladus.model.conformanceRule.ConformanceRule import za.co.absa.enceladus.model.properties.PropertyDefinition +import za.co.absa.enceladus.model.versionedModel.VersionedSummary import za.co.absa.enceladus.model.{Dataset, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel.Constants.DefaultValidationLevelName @@ -44,6 +46,14 @@ class DatasetController @Autowired()(datasetService: DatasetService) import scala.concurrent.ExecutionContext.Implicits.global + @GetMapping(Array("/latest")) + @ResponseStatus(HttpStatus.OK) + def getLatestVersions(@RequestParam(value = "missing_property", required = false) + missingProperty: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = { + datasetService.getLatestVersions(missingProperty.toScalaOption) + .map(datasets => datasets.map(dataset => VersionedSummary(dataset.name, dataset.version))) + } + @PostMapping(Array("/{datasetName}/rule/create")) @ResponseStatus(HttpStatus.OK) def addConformanceRule(@AuthenticationPrincipal user: UserDetails, diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala index e372b5be8..b7832d706 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala @@ -18,20 +18,19 @@ package za.co.absa.enceladus.menas.controllers import java.util.concurrent.CompletableFuture import scala.concurrent.Future - import org.springframework.beans.factory.annotation.Autowired import org.springframework.scheduling.annotation.Async import org.springframework.scheduling.annotation.Scheduled import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RestController - import za.co.absa.enceladus.menas.models.LandingPageInformation import za.co.absa.enceladus.menas.repositories.DatasetMongoRepository import za.co.absa.enceladus.menas.repositories.LandingPageStatisticsMongoRepository import za.co.absa.enceladus.menas.repositories.MappingTableMongoRepository import za.co.absa.enceladus.menas.repositories.SchemaMongoRepository -import za.co.absa.enceladus.menas.services.RunService +import za.co.absa.enceladus.menas.services.{PropertyDefinitionService, RunService, StatisticsService} +import za.co.absa.enceladus.model.properties.essentiality.{Mandatory, Recommended} @RestController @RequestMapping(Array("/api/landing")) @@ -39,7 +38,8 @@ class LandingPageController @Autowired() (datasetRepository: DatasetMongoReposit mappingTableRepository: MappingTableMongoRepository, schemaRepository: SchemaMongoRepository, runsService: RunService, - landingPageRepository: LandingPageStatisticsMongoRepository) extends BaseController { + landingPageRepository: LandingPageStatisticsMongoRepository, + statisticsService: StatisticsService) extends BaseController { import scala.concurrent.ExecutionContext.Implicits.global import za.co.absa.enceladus.menas.utils.implicits._ @@ -50,13 +50,31 @@ class LandingPageController @Autowired() (datasetRepository: DatasetMongoReposit } def landingPageInfo(): Future[LandingPageInformation] = { + val dsCountFuture = datasetRepository.distinctCount() + val mappingTableFuture = mappingTableRepository.distinctCount() + val schemaFuture = schemaRepository.distinctCount() + val runFuture = runsService.getCount() + val propertiesWithMissingCountsFuture = statisticsService.getPropertiesWithMissingCount() + val propertiesTotalsFuture: Future[(Int, Int, Int)] = propertiesWithMissingCountsFuture.map(props => { + props.foldLeft(0, 0, 0) { (acum, item) => + val (count, mandatoryCount, recommendedCount) = acum + item.essentiality match { + case Mandatory(_) => (count + 1, mandatoryCount + item.missingInDatasetsCount, recommendedCount) + case Recommended() => (count + 1, mandatoryCount, recommendedCount + item.missingInDatasetsCount) + case _ => (count + 1, mandatoryCount, recommendedCount) + } + } + }) + val todaysStatsfuture = runsService.getTodaysRunsStatistics() for { - dsCount <- datasetRepository.distinctCount() - mtCount <- mappingTableRepository.distinctCount() - schemaCount <- schemaRepository.distinctCount() - runCount <- runsService.getCount() - todaysStats <- runsService.getTodaysRunsStatistics() - } yield LandingPageInformation(dsCount, mtCount, schemaCount, runCount, todaysStats) + dsCount <- dsCountFuture + mtCount <- mappingTableFuture + schemaCount <- schemaFuture + runCount <- runFuture + (propertiesCount, totalMissingMandatoryProperties, totalMissingRecommendedProperties) <- propertiesTotalsFuture + todaysStats <- todaysStatsfuture + } yield LandingPageInformation(dsCount, mtCount, schemaCount, runCount, propertiesCount, + totalMissingMandatoryProperties, totalMissingRecommendedProperties, todaysStats) } // scalastyle:off magic.number @@ -67,6 +85,6 @@ class LandingPageController @Autowired() (datasetRepository: DatasetMongoReposit for { newStats <- landingPageInfo() res <- landingPageRepository.updateStatistics(newStats) - } yield res + } yield res } } diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/StatisticsController.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/StatisticsController.scala new file mode 100644 index 000000000..492d5d303 --- /dev/null +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/StatisticsController.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.menas.controllers + +import java.util.concurrent.CompletableFuture + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.bind.annotation.{GetMapping, RequestMapping, RestController} +import za.co.absa.enceladus.menas.services.StatisticsService +import za.co.absa.enceladus.model.properties.PropertyDefinitionStats + +@RestController +@RequestMapping(Array("/api/statistics")) +class StatisticsController @Autowired() (statisticsService: StatisticsService) extends BaseController { + + import za.co.absa.enceladus.menas.utils.implicits._ + + @GetMapping(Array("/properties/missing")) + def getPropertiesWithMissingCount(): CompletableFuture[Seq[PropertyDefinitionStats]] = { + statisticsService.getPropertiesWithMissingCount() + } + +} diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/models/LandingPageInformation.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/models/LandingPageInformation.scala index 5950e1ded..c4c830e4b 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/models/LandingPageInformation.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/models/LandingPageInformation.scala @@ -20,4 +20,7 @@ case class LandingPageInformation( totalNumberMappingTables: Int, totalNumberSchemas: Int, totalNumberRuns: Long, + totalNumberProperties: Int, + totalNumberMissingMandatoryProperties: Int, + totalNumberMissingRecommendedProperties: Int, todaysRunsStatistics: TodaysRunsStatistics) diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/repositories/VersionedMongoRepository.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/repositories/VersionedMongoRepository.scala index 3fbce015a..bbf4822b8 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/repositories/VersionedMongoRepository.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/repositories/VersionedMongoRepository.scala @@ -75,15 +75,10 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab collection.aggregate[VersionedSummary](pipeline).toFuture() } - def getLatestVersions(): Future[Seq[C]] = { - // there may be a way to this using mongo-joining (aggregation.lookup) instead - getLatestVersionsSummary(None).flatMap { summaries => - val resultIn = summaries.map { summary => - getVersion(summary._id, summary.latestVersion).map(_.toSeq) - } - - Future.sequence(resultIn).map(_.flatten) - } + def getLatestVersions(missingProperty: Option[String]): Future[Seq[C]] = { + val missingFilter = missingProperty.map(missingProp => + Filters.not(Filters.exists(s"properties.$missingProp"))) + collectLatestVersions(missingFilter) } def getVersion(name: String, version: Int): Future[Option[C]] = { @@ -163,6 +158,18 @@ abstract class VersionedMongoRepository[C <: VersionedModel](mongoDb: MongoDatab .toFuture() } + private def collectLatestVersions(postAggFilter: Option[Bson]): Future[Seq[C]] = { + val pipeline = Seq( + filter(Filters.notEqual("disabled", true)), + Aggregates.group("$name", + Accumulators.max("latestVersion", "$version"), + Accumulators.last("doc","$$ROOT")), + Aggregates.replaceRoot("$doc")) ++ + postAggFilter.map(Aggregates.filter) + + collection.aggregate[C](pipeline).toFuture() + } + private[repositories] def getNotDisabledFilter: Bson = { notEqual("disabled", true) } diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala index 40527fea1..b7d9a8835 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala @@ -15,24 +15,21 @@ package za.co.absa.enceladus.menas.services -import scala.concurrent.Future import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.menas.repositories.DatasetMongoRepository -import za.co.absa.enceladus.menas.repositories.OozieRepository -import za.co.absa.enceladus.model.{Dataset, Schema, UsedIn, Validation} +import za.co.absa.enceladus.menas.repositories.{DatasetMongoRepository, OozieRepository} +import za.co.absa.enceladus.menas.services.DatasetService.{RuleValidationsAndFields, _} import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, _} import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieScheduleInstance - -import scala.language.reflectiveCalls -import DatasetService.RuleValidationsAndFields -import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel import za.co.absa.enceladus.model.properties.PropertyDefinition import za.co.absa.enceladus.model.properties.essentiality.Essentiality._ import za.co.absa.enceladus.model.properties.essentiality.Mandatory +import za.co.absa.enceladus.model.{Dataset, Schema, UsedIn, Validation} import za.co.absa.enceladus.utils.validation.ValidationLevel -import DatasetService._ +import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel +import scala.concurrent.Future +import scala.language.reflectiveCalls import scala.util.{Failure, Success} @@ -217,6 +214,9 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository } } + def getLatestVersions(missingProperty: Option[String]): Future[Seq[Dataset]] = + datasetMongoRepository.getLatestVersions(missingProperty) + override def importItem(item: Dataset, username: String): Future[Option[Dataset]] = { getLatestVersionValue(item.name).flatMap { case Some(version) => update(username, item.copy(version = version)) diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/PropertyDefinitionService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/PropertyDefinitionService.scala index 2041c3f45..a7851d542 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/services/PropertyDefinitionService.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/PropertyDefinitionService.scala @@ -17,8 +17,7 @@ package za.co.absa.enceladus.menas.services import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import za.co.absa.enceladus.menas.repositories.{DatasetMongoRepository, PropertyDefinitionMongoRepository} -import za.co.absa.enceladus.menas.utils.converters.SparkMenasSchemaConvertor +import za.co.absa.enceladus.menas.repositories.PropertyDefinitionMongoRepository import za.co.absa.enceladus.model.UsedIn import za.co.absa.enceladus.model.properties.PropertyDefinition @@ -42,6 +41,10 @@ class PropertyDefinitionService @Autowired()(propertyDefMongoRepository: Propert } } + def getDistinctCount(): Future[Int] = { + propertyDefMongoRepository.distinctCount() + } + override def create(newPropertyDef: PropertyDefinition, username: String): Future[Option[PropertyDefinition]] = { val propertyDefBase = PropertyDefinition( name = newPropertyDef.name, diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/StatisticsService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/StatisticsService.scala new file mode 100644 index 000000000..9f55f4d21 --- /dev/null +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/StatisticsService.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.menas.services + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component +import za.co.absa.enceladus.model.properties.{PropertyDefinition, PropertyDefinitionStats} + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +@Component +class StatisticsService @Autowired() (propertyDefService: PropertyDefinitionService, datasetService: DatasetService){ + //#TODO find optimizations #1897 + def getPropertiesWithMissingCount(): Future[Seq[PropertyDefinitionStats]] = { + val propertyDefsFuture = propertyDefService.getLatestVersions() + propertyDefsFuture + .map { (props: Seq[PropertyDefinition]) => + val propertiesWithMissingCounts: Seq[Future[PropertyDefinitionStats]] = props.map(propertyDef => + datasetService + .getLatestVersions(Some(propertyDef.name)) + .map(datasetsMissingProp => + PropertyDefinitionStats(propertyDef, datasetsMissingProp.size)) + ) + propertiesWithMissingCounts + } + .flatMap { propertiesWithMissingCounts: Seq[Future[PropertyDefinitionStats]] => + Future.sequence(propertiesWithMissingCounts) + } + } + +} diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/VersionedModelService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/VersionedModelService.scala index a486492d5..7f7b4954c 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/services/VersionedModelService.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/VersionedModelService.scala @@ -41,7 +41,7 @@ abstract class VersionedModelService[C <: VersionedModel with Product with Audit } def getLatestVersions(): Future[Seq[C]] = { - versionedMongoRepository.getLatestVersions() + versionedMongoRepository.getLatestVersions(None) } def getSearchSuggestions(): Future[Seq[String]] = { diff --git a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala index fbd5a8346..e81d14694 100644 --- a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala +++ b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/controllers/PropertyDefinitionApiIntegrationSuite.scala @@ -384,7 +384,7 @@ class PropertyDefinitionApiIntegrationSuite extends BaseRestApiTest with BeforeA val response = sendGet[Array[PropertyDefinition]](s"$apiUrl") // Array to avoid erasure assertOk(response) - val responseData = response.getBody.toSeq.map(pd => (pd.name, pd.version)) + val responseData = response.getBody.toSeq.map(pd => (pd.name, pd.version)).sortBy(_._1) val expectedData = Seq("propertyDefinitionA" -> 2, "propertyDefinitionB" -> 3) // disabled pdA-v3 not reported assert(responseData == expectedData) } diff --git a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/DatasetRepositoryIntegrationSuite.scala b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/DatasetRepositoryIntegrationSuite.scala index eb8838781..2f5be1aee 100644 --- a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/DatasetRepositoryIntegrationSuite.scala +++ b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/DatasetRepositoryIntegrationSuite.scala @@ -24,6 +24,7 @@ import org.springframework.test.context.junit4.SpringRunner import za.co.absa.enceladus.menas.exceptions.EntityAlreadyExistsException import za.co.absa.enceladus.menas.integration.fixtures.{DatasetFixtureService, FixtureService} import za.co.absa.enceladus.menas.repositories.DatasetMongoRepository +import za.co.absa.enceladus.model.Dataset import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, MappingConformanceRule} import za.co.absa.enceladus.model.test.factories.DatasetFactory import za.co.absa.enceladus.model.menas.scheduler.oozie.OozieSchedule @@ -512,6 +513,27 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { val expected = Seq(dataset3, dataset4).map(DatasetFactory.toSummary) assert(actual == expected) } + + "search with missing properties" in { + val dataset1ver1 = DatasetFactory.getDummyDataset(name = "dataset1", version = 1) + val dataset1ver2 = DatasetFactory.getDummyDataset(name = "dataset1", version = 2, + properties = Some(Map("prop1"->"a"))) + val dataset2ver1 = DatasetFactory.getDummyDataset(name = "dataset2", version = 1) + val dataset2ver2 = DatasetFactory.getDummyDataset(name = "dataset2", version = 2) + val dataset3ver1 = DatasetFactory.getDummyDataset(name = "dataset3", version = 1) + val dataset4ver1 = DatasetFactory.getDummyDataset(name = "dataset4", version = 1, + properties = Some(Map("prop1"->"A"))) + + val abc1 = DatasetFactory.getDummyDataset(name = "abc", version = 1) + + datasetFixture.add(dataset1ver1, dataset1ver2, dataset2ver1, dataset2ver2, dataset3ver1, dataset4ver1, abc1) + + val actual: Seq[Dataset] = await(datasetMongoRepository.getLatestVersions(Some("prop1"))) + .sortBy(_.name) + + val expected = Seq(abc1, dataset2ver2, dataset3ver1) + assert(actual == expected) + } } "return all datasets" when { @@ -595,7 +617,7 @@ class DatasetRepositoryIntegrationSuite extends BaseRepositoryTest { assert(await(datasetMongoRepository.findByCoordId("SomeCoordId")) == Seq()) } } - "return datasets witch matching coordinator ID" when { + "return datasets with matching coordinator ID" when { "such datasets exist" in { val schedule = OozieSchedule(scheduleTiming = ScheduleTiming(Seq(), Seq(), Seq(), Seq(), Seq()), runtimeParams = RuntimeConfig(sysUser = "user", menasKeytabFile = "/a/b/c"), datasetVersion = 0, diff --git a/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/StatisticsIntegrationSuite.scala b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/StatisticsIntegrationSuite.scala new file mode 100644 index 000000000..69356fc11 --- /dev/null +++ b/menas/src/test/scala/za/co/absa/enceladus/menas/integration/repositories/StatisticsIntegrationSuite.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.enceladus.menas.integration.repositories + +import org.junit.runner.RunWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.junit4.SpringRunner +import za.co.absa.enceladus.menas.integration.fixtures.{DatasetFixtureService, FixtureService, PropertyDefinitionFixtureService} +import za.co.absa.enceladus.menas.repositories.{DatasetMongoRepository, PropertyDefinitionMongoRepository} +import za.co.absa.enceladus.menas.services.StatisticsService +import za.co.absa.enceladus.model.properties.{PropertyDefinition, PropertyDefinitionStats} +import za.co.absa.enceladus.model.properties.essentiality.Essentiality.{Mandatory, Optional, Recommended} +import za.co.absa.enceladus.model.properties.propertyType.{EnumPropertyType, StringPropertyType} +import za.co.absa.enceladus.model.test.factories.DatasetFactory + +@RunWith(classOf[SpringRunner]) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(Array("withEmbeddedMongo")) +class StatisticsIntegrationSuite extends BaseRepositoryTest { + @Autowired + private val datasetFixture: DatasetFixtureService = null + + @Autowired + private val propertyDefService: PropertyDefinitionFixtureService = null + + @Autowired + private val statisticsService: StatisticsService = null + + override def fixtures: List[FixtureService[_]] = List(datasetFixture) + + val mockedPropertyDefinitions = Seq( + PropertyDefinition(name = "mandatoryString1", propertyType = StringPropertyType(), essentiality = Mandatory(allowRun = false), + userCreated = "", userUpdated = ""), + PropertyDefinition(name = "mandatoryString2", propertyType = StringPropertyType(), essentiality = Mandatory(allowRun = false), + userCreated = "", userUpdated = ""), + PropertyDefinition(name = "recommendedString1", propertyType = StringPropertyType(), essentiality = Recommended, + userCreated = "", userUpdated = ""), + PropertyDefinition(name = "optionalString1", propertyType = StringPropertyType(), essentiality = Optional, + userCreated = "", userUpdated = ""), + PropertyDefinition(name = "mandatoryDisabledString1", propertyType = StringPropertyType(), essentiality = Mandatory(allowRun = false), + disabled = true, userCreated = "", userUpdated = ""), + PropertyDefinition(name = "optionalEnumAb", propertyType = EnumPropertyType("optionA", "optionB"), essentiality = Optional, + userCreated = "", userUpdated = "") + ) + + val mockedDatasets = Seq( + DatasetFactory.getDummyDataset(name = "dataset1", version = 1, properties = Some( + Map())), + DatasetFactory.getDummyDataset(name = "dataset1", version = 2, properties = Some( + Map("mandatoryString1"->""))), + DatasetFactory.getDummyDataset(name = "dataset1", version = 3, properties = Some( + Map("mandatoryString1"->"", "mandatoryString2"->"3", "optionalEnumAb" -> "optionA"))), + DatasetFactory.getDummyDataset(name = "dataset2", version = 1, properties = Some( + Map("recommendedString1" -> "", "optionalString1"->""))), + DatasetFactory.getDummyDataset(name = "dataset2", version = 2, properties = Some( + Map("mandatoryString1"->"", "recommendedString1" -> ""))), + DatasetFactory.getDummyDataset(name = "dataset3", version = 1, properties = Some( + Map("mandatoryString1"->""))), + DatasetFactory.getDummyDataset(name = "dataset3", version = 2, properties = Some( + Map("mandatoryString1"->"","mandatoryString2"->"3"))), + DatasetFactory.getDummyDataset(name = "dataset4", version = 1, properties = Some( + Map("mandatoryString1"->"", "mandatoryString2"->"3", "recommendedString1" -> "", "optionalString1"->"", + "mandatoryDisabledString1" -> "", "optionalEnumAb" -> "optionA"))), + DatasetFactory.getDummyDataset(name = "dataset5", version = 1, properties = Some(Map())), + DatasetFactory.getDummyDataset(name = "dataset6", version = 1, properties = Some(Map( + "mandatoryString1"->"", "mandatoryString2"->"3", "recommendedString1" -> ""))) + ) + + "StatisticsService" should { + + "return the properties with missing counts" when { + "the specified datasets and properties" in { + datasetFixture.add(mockedDatasets: _*) + propertyDefService.add(mockedPropertyDefinitions: _*) + + val actualStatistics = await(statisticsService.getPropertiesWithMissingCount()).sortBy(_.name) + + val expectedStatistics = Seq( + PropertyDefinitionStats(name = "mandatoryString1", essentiality = Mandatory(allowRun = false), + missingInDatasetsCount = 1), // missing in dataset5 + PropertyDefinitionStats(name = "mandatoryString2", essentiality = Mandatory(allowRun = false), + missingInDatasetsCount = 2), // missing in dataset2,5 + PropertyDefinitionStats(name = "optionalEnumAb", essentiality = Optional, + missingInDatasetsCount = 4), // missing in dataset2,3,5,6 + PropertyDefinitionStats(name = "optionalString1", essentiality = Optional, + missingInDatasetsCount = 5), // missing in dataset1,2,3,5,6 + PropertyDefinitionStats(name = "recommendedString1", essentiality = Recommended, + missingInDatasetsCount = 3) // missing in dataset1,3,5 + ) + + assert(actualStatistics == expectedStatistics) + } + } + } +} From 758ac7d11883ff2f56d43e4fcf0a7fde9d8346cc Mon Sep 17 00:00:00 2001 From: Adrian Olosutean Date: Wed, 1 Sep 2021 13:12:22 +0300 Subject: [PATCH 05/21] 1843 Summary page for properties (#1880) * 1843 Home page with properties, side panel with missing counts and summary page for properties with tab containing datasets missing that particular property --- .../controllers/LandingPageController.scala | 2 +- menas/ui/components/Component.js | 12 ++- menas/ui/components/app.controller.js | 5 ++ menas/ui/components/app.view.xml | 2 + .../components/home/landingPage.controller.js | 2 + menas/ui/components/home/landingPage.view.xml | 26 +++++++ .../datasetPropertyDetail.controller.js | 73 +++++++++++++++++++ .../property/datasetPropertyDetail.view.xml | 58 +++++++++++++++ .../property/datasetPropertyInfo.fragment.xml | 45 ++++++++++++ .../datasetPropertyMaster.controller.js | 51 +++++++++++++ .../property/datasetPropertyMaster.view.xml | 33 +++++++++ menas/ui/service/EntityService.js | 36 +++++++++ menas/ui/service/MessageProvider.js | 8 ++ menas/ui/service/PropertiesService.js | 4 + menas/ui/service/RestDAO.js | 15 ++++ 15 files changed, 370 insertions(+), 2 deletions(-) create mode 100644 menas/ui/components/property/datasetPropertyDetail.controller.js create mode 100644 menas/ui/components/property/datasetPropertyDetail.view.xml create mode 100644 menas/ui/components/property/datasetPropertyInfo.fragment.xml create mode 100644 menas/ui/components/property/datasetPropertyMaster.controller.js create mode 100644 menas/ui/components/property/datasetPropertyMaster.view.xml diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala index b7832d706..e2a8f9136 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/controllers/LandingPageController.scala @@ -29,7 +29,7 @@ import za.co.absa.enceladus.menas.repositories.DatasetMongoRepository import za.co.absa.enceladus.menas.repositories.LandingPageStatisticsMongoRepository import za.co.absa.enceladus.menas.repositories.MappingTableMongoRepository import za.co.absa.enceladus.menas.repositories.SchemaMongoRepository -import za.co.absa.enceladus.menas.services.{PropertyDefinitionService, RunService, StatisticsService} +import za.co.absa.enceladus.menas.services.{RunService, StatisticsService} import za.co.absa.enceladus.model.properties.essentiality.{Mandatory, Recommended} @RestController diff --git a/menas/ui/components/Component.js b/menas/ui/components/Component.js index a6f3278f2..c5ea51927 100644 --- a/menas/ui/components/Component.js +++ b/menas/ui/components/Component.js @@ -70,7 +70,12 @@ sap.ui.define([ name: "mappingTables", pattern: "mapping/:id:/:version:", target: "mappingTable" - } + }, + { + name: "properties", + pattern: "properties/:id:", + target: "property" + }, ], targets: { login: { @@ -102,6 +107,11 @@ sap.ui.define([ viewName: "components.mappingTable.mappingTableDetail", viewLevel: 1, viewId: "mappingTableDetailView" + }, + property: { + viewName: "components.property.datasetPropertyDetail", + viewLevel: 1, + viewId: "datasetPropertyDetailView" } } } diff --git a/menas/ui/components/app.controller.js b/menas/ui/components/app.controller.js index 67c4f7ec9..4f7949ec4 100644 --- a/menas/ui/components/app.controller.js +++ b/menas/ui/components/app.controller.js @@ -129,6 +129,11 @@ sap.ui.define([ this._app.toMaster(this.createId("mappingTablesPage")); }, + onPropertiesPress: function (oEv) { + this._eventBus.publish("properties", "list"); + this._app.toMaster(this.createId("propertiesPage")); + }, + onEntityCreated: function (sTopic, sEvent, oData) { this._router.navTo(sTopic, { id: oData.name, diff --git a/menas/ui/components/app.view.xml b/menas/ui/components/app.view.xml index 46419866c..fc779d087 100644 --- a/menas/ui/components/app.view.xml +++ b/menas/ui/components/app.view.xml @@ -25,6 +25,7 @@ +