From 741d316552484a532b6927cf6bfbb8fc0e29983b Mon Sep 17 00:00:00 2001 From: Gary Luo Date: Wed, 15 Mar 2017 16:26:51 -0500 Subject: [PATCH] 1. Implement QueryPlanning and DataSource Constraints into Resolvers and Matchers (#169) --- CHANGELOG.md | 6 + .../bard/webservice/async/AsyncUtils.java | 6 +- .../webservice/data/DruidQueryBuilder.java | 8 +- .../webservice/data/PartialDataHandler.java | 14 +-- .../data/metric/mappers/RowNumMapper.java | 1 - .../model/datasource/TableDataSource.java | 2 +- .../AggregatableDimensionsMatcher.java | 24 ++-- .../resolver/BasePhysicalTableResolver.java | 96 ++++------------ .../table/resolver/DataSourceConstraint.java | 88 +++++++++++++++ .../DefaultPhysicalTableResolver.java | 27 ++--- .../table/resolver/PartialTimeComparator.java | 29 ++--- .../table/resolver/PhysicalTableResolver.java | 10 +- .../resolver/QueryPlanningConstraint.java | 70 ++++++++++++ .../resolver/SchemaPhysicalTableMatcher.java | 44 +++----- .../TimeAlignmentPhysicalTableMatcher.java | 15 ++- .../resolver/VolatileTimeComparator.java | 28 ++--- .../handlers/PartialDataRequestHandler.java | 6 +- .../data/PartialDataHandlerSpec.groovy | 66 +++-------- ...DimensionToDefaultDimensionSpecSpec.groovy | 1 + .../LookupDimensionToDimensionSpecSpec.groovy | 1 + ...dLookupDimensionToDimensionSpecSpec.groovy | 1 + .../BasePhysicalTableResolverSpec.groovy | 106 +++++++----------- .../DefaultPhysicalTableResolverSpec.groovy | 37 +++--- .../SchemaPhysicalTableMatcherSpec.groovy | 18 +-- .../VolatileTimeComparatorSpec.groovy | 17 +-- .../PartialDataRequestHandlerSpec.groovy | 19 ++-- 26 files changed, 377 insertions(+), 363 deletions(-) create mode 100644 fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/DataSourceConstraint.java create mode 100644 fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/QueryPlanningConstraint.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b44a74c9f0..d2fdb17327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ Current ------- ### Added: +- [QueryPlanningConstraint and DataSourceConstraint](https://github.com/yahoo/fili/pull/169) + * Added `QueryPlanningConstraint` to replace current interface of Matchers and Resolvers arguments during query planning + * Added `DataSourceConstraint` to allow implementation of `PartitionedFactTable`'s availability in the near future - [Major refactor for availability and schemas and tables](https://github.com/yahoo/fili/pull/165) * `ImmutableAvailability` - provides immutable, typed replacement for maps of column availabilities @@ -43,6 +46,9 @@ Current - [Support timeouts for lucene search provider](https://github.com/yahoo/fili/pull/183) ### Changed: +- [QueryPlanningConstraint and DataSourceConstraint](https://github.com/yahoo/fili/pull/169) + * `QueryPlanningConstraint` replaces current interface of Matchers and Resolvers `DataApiRequest` and `TemplateDruidQuery` arguments during query planning + * Modified `findMissingTimeGrainIntervals` method in `PartialDataHandler` to take a set of columns instead of `DataApiRequest` and `DruidAggregationQuery` - [Major refactor for availability and schemas and tables](https://github.com/yahoo/fili/pull/165) * `Schema` and `Table` became interfaces diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/async/AsyncUtils.java b/fili-core/src/main/java/com/yahoo/bard/webservice/async/AsyncUtils.java index ca3fe29f51..3337902897 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/async/AsyncUtils.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/async/AsyncUtils.java @@ -61,10 +61,8 @@ public static PreResponse buildErrorPreResponse(Throwable throwable) { } return new PreResponse( - new ResultSet(new ResultSetSchema( - AllGranularity.INSTANCE, - Collections.emptySet()), - Collections.emptyList() + new ResultSet( + new ResultSetSchema(AllGranularity.INSTANCE, Collections.emptySet()), Collections.emptyList() ), responseContext ); diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/DruidQueryBuilder.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/DruidQueryBuilder.java index 93d79ce44a..1db27b5d38 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/DruidQueryBuilder.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/DruidQueryBuilder.java @@ -1,4 +1,4 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.data; @@ -28,6 +28,7 @@ import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.table.TableGroup; import com.yahoo.bard.webservice.table.TableIdentifier; +import com.yahoo.bard.webservice.table.resolver.QueryPlanningConstraint; import com.yahoo.bard.webservice.table.resolver.NoMatchFoundException; import com.yahoo.bard.webservice.table.resolver.PhysicalTableResolver; import com.yahoo.bard.webservice.web.DataApiRequest; @@ -118,7 +119,10 @@ public DruidAggregationQuery buildQuery( TableGroup group = logicalTable.getTableGroup(); // Resolve the table from the the group, the combined dimensions in request, and template time grain - PhysicalTable table = resolver.resolve(group.getPhysicalTables(), request, template); + PhysicalTable table = resolver.resolve( + group.getPhysicalTables(), + new QueryPlanningConstraint(request, template) + ); return druidTopNMetric != null ? buildTopNQuery( diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/PartialDataHandler.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/PartialDataHandler.java index f2ecf6e341..265bee588c 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/PartialDataHandler.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/PartialDataHandler.java @@ -1,4 +1,4 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.data; @@ -61,8 +61,7 @@ public SimplifiedIntervalList findMissingRequestTimeGrainIntervals( throw new IllegalArgumentException(message); } return findMissingTimeGrainIntervals( - apiRequest, - query, + TableUtils.getColumnNames(apiRequest, query), physicalTables, new SimplifiedIntervalList(apiRequest.getIntervals()), apiRequest.getGranularity() @@ -83,8 +82,7 @@ public SimplifiedIntervalList findMissingRequestTimeGrainIntervals( * present for a given combination of request metrics and dimensions (pulled from the API request and generated * druid query) at the specified granularity. * - * @param apiRequest api request made by the end user - * @param query used to fetch data from druid + * @param columnNames all the column names the request depends on * @param physicalTables the tables whose column availabilities are checked * @param requestedIntervals The intervals that may not be fully satisfied * @param granularity The granularity at which to find missing intervals @@ -92,14 +90,13 @@ public SimplifiedIntervalList findMissingRequestTimeGrainIntervals( * @return subintervals of the requested intervals with incomplete data */ public SimplifiedIntervalList findMissingTimeGrainIntervals( - DataApiRequest apiRequest, - DruidAggregationQuery query, + Set columnNames, Set physicalTables, @NotNull SimplifiedIntervalList requestedIntervals, Granularity granularity ) { SimplifiedIntervalList availableIntervals = physicalTables.stream() - .map(table -> getAvailability(table, TableUtils.getColumnNames(apiRequest, query))) + .map(table -> getAvailability(table, columnNames)) .flatMap(SimplifiedIntervalList::stream) .collect(SimplifiedIntervalList.getCollector()); @@ -108,6 +105,7 @@ public SimplifiedIntervalList findMissingTimeGrainIntervals( requestedIntervals, granularity ); + if (granularity instanceof AllGranularity && !missingIntervals.isEmpty()) { missingIntervals = requestedIntervals; } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/metric/mappers/RowNumMapper.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/metric/mappers/RowNumMapper.java index 4fc771a5de..d3ec5c7714 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/metric/mappers/RowNumMapper.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/metric/mappers/RowNumMapper.java @@ -43,7 +43,6 @@ public ResultSet map(ResultSet resultSet) { @Override protected Result map(Result result, ResultSetSchema schema) { - // map for rows is not throw new UnsupportedOperationException("This code should never be reached."); } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/datasource/TableDataSource.java b/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/datasource/TableDataSource.java index c135aa35fb..36b9a74881 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/datasource/TableDataSource.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/datasource/TableDataSource.java @@ -25,7 +25,7 @@ public class TableDataSource extends DataSource { public TableDataSource(ConcretePhysicalTable physicalTable) { super(DefaultDataSourceType.TABLE, Collections.singleton(physicalTable)); - this.name = physicalTable.getFactTableName(); + this.name = physicalTable.getAvailability().getDataSourceNames().stream().findFirst().get().asName(); } public String getName() { diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/AggregatableDimensionsMatcher.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/AggregatableDimensionsMatcher.java index 11b863e7f9..1c7aed0fe7 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/AggregatableDimensionsMatcher.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/AggregatableDimensionsMatcher.java @@ -1,15 +1,12 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver; import static com.yahoo.bard.webservice.web.ErrorMessageFormat.NO_TABLE_FOR_NON_AGGREGATABLE; -import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.data.dimension.Dimension; -import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery; +import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.util.StreamUtils; -import com.yahoo.bard.webservice.util.TableUtils; -import com.yahoo.bard.webservice.web.DataApiRequest; import com.yahoo.bard.webservice.web.ErrorMessageFormat; import org.slf4j.Logger; @@ -29,23 +26,20 @@ public class AggregatableDimensionsMatcher implements PhysicalTableMatcher { public static final ErrorMessageFormat MESSAGE_FORMAT = NO_TABLE_FOR_NON_AGGREGATABLE; - private final DataApiRequest request; - private final TemplateDruidQuery query; + private final QueryPlanningConstraint requestConstraints; /** * Constructor saves metrics, dimensions, coarsest time grain, and logical table name (for logging). * - * @param request The request whose dimensions are being matched on - * @param query The query whose columns are being matched + * @param requestConstraints Contains the request constraints extracted from DataApiRequest and TemplateDruidQuery */ - public AggregatableDimensionsMatcher(DataApiRequest request, TemplateDruidQuery query) { - this.request = request; - this.query = query; + public AggregatableDimensionsMatcher(QueryPlanningConstraint requestConstraints) { + this.requestConstraints = requestConstraints; } @Override public boolean test(PhysicalTable table) { - Set columnNames = TableUtils.getColumnNames(request, query.getInnermostQuery()); + Set columnNames = requestConstraints.getAllColumnNames(); // If table contains non-agg dimensions, query must contain all these non-agg dimensions to use this table. return table.getDimensions().stream() @@ -56,12 +50,12 @@ public boolean test(PhysicalTable table) { @Override public NoMatchFoundException noneFoundException() { - Set aggDimensions = request.getDimensions().stream() + Set aggDimensions = requestConstraints.getRequestDimensions().stream() .filter(Dimension::isAggregatable) .map(Dimension::getApiName) .collect(Collectors.toSet()); - Set nonAggDimensions = request.getDimensions().stream() + Set nonAggDimensions = requestConstraints.getRequestDimensions().stream() .filter(StreamUtils.not(Dimension::isAggregatable)) .map(Dimension::getApiName) .collect(Collectors.toSet()); diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/BasePhysicalTableResolver.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/BasePhysicalTableResolver.java index 7072d21fc2..295094cfeb 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/BasePhysicalTableResolver.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/BasePhysicalTableResolver.java @@ -1,15 +1,9 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver; -import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.application.MetricRegistryFactory; -import com.yahoo.bard.webservice.data.dimension.Dimension; -import com.yahoo.bard.webservice.data.metric.LogicalMetric; -import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery; -import com.yahoo.bard.webservice.druid.model.query.Granularity; -import com.yahoo.bard.webservice.util.TableUtils; -import com.yahoo.bard.webservice.web.DataApiRequest; +import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.web.ErrorMessageFormat; import com.codahale.metrics.MetricRegistry; @@ -21,9 +15,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import java.util.function.BiFunction; import java.util.function.BinaryOperator; -import java.util.stream.Collectors; /** * Abstract parent to with business rule agnostic implementations of core methods. @@ -33,44 +25,29 @@ public abstract class BasePhysicalTableResolver implements PhysicalTableResolver private static final Logger LOG = LoggerFactory.getLogger(BasePhysicalTableResolver.class); private static final MetricRegistry REGISTRY = MetricRegistryFactory.getRegistry(); - protected final BiFunction resolveAcceptingGrain; - - /** - * Constructor. - */ - public BasePhysicalTableResolver() { - this.resolveAcceptingGrain = new RequestQueryGranularityResolver(); - } - /** * Create a list of matchers based on a request and query. * - * @param apiRequest The ApiRequest for the query - * @param query a partial query representation + * @param requestConstraints contains the request constraints extracted from DataApiRequest and TemplateDruidQuery * * @return a list of matchers to be applied, in order */ - public abstract List getMatchers(DataApiRequest apiRequest, TemplateDruidQuery query); + public abstract List getMatchers(QueryPlanningConstraint requestConstraints); /** * Create a binary operator which returns the 'better' of two physical table. * - * @param apiRequest The ApiRequest for the query - * @param query a partial query representation + * @param requestConstraints contains the request constraints extracted from DataApiRequest and TemplateDruidQuery * * @return a list of matchers to be applied, in order */ - public abstract BinaryOperator getBetterTableOperator( - DataApiRequest apiRequest, - TemplateDruidQuery query - ); + public abstract BinaryOperator getBetterTableOperator(QueryPlanningConstraint requestConstraints); /** * Filter to a set of tables matching the rules of this resolver. * * @param candidateTables The physical tables being filtered - * @param apiRequest The request being filtered to - * @param query a partial query representation + * @param requestConstraints contains the request constraints extracted from DataApiRequest and TemplateDruidQuery * * @return a set of physical tables which all match the criteria of a request and partial query * @@ -78,10 +55,9 @@ public abstract BinaryOperator getBetterTableOperator( */ public Set filter( Collection candidateTables, - DataApiRequest apiRequest, - TemplateDruidQuery query + QueryPlanningConstraint requestConstraints ) throws NoMatchFoundException { - return filter(candidateTables, getMatchers(apiRequest, query)); + return filter(candidateTables, getMatchers(requestConstraints)); } /** @@ -108,64 +84,34 @@ public Set filter( @Override public PhysicalTable resolve( Collection candidateTables, - DataApiRequest apiRequest, - TemplateDruidQuery query + QueryPlanningConstraint requestConstraints ) throws NoMatchFoundException { + // Minimum grain at which the request can be aggregated from - Granularity minimumTableTimeGrain = resolveAcceptingGrain.apply(apiRequest, query); - TemplateDruidQuery innerQuery = (TemplateDruidQuery) query.getInnermostQuery(); - Set columnNames = TableUtils.getDimensions(apiRequest, innerQuery) - .map(Dimension::getApiName) - .collect(Collectors.toSet()); LOG.trace( "Resolving Table using TimeGrain: {}, dimension API names: {} and TableGroup: {}", - minimumTableTimeGrain, - columnNames, + requestConstraints.getMinimumGranularity(), + requestConstraints.getAllColumnNames(), candidateTables ); try { - Set physicalTables = filter(candidateTables, apiRequest, query); + Set physicalTables = filter(candidateTables, requestConstraints); - BinaryOperator betterTable = getBetterTableOperator(apiRequest, query); - PhysicalTable bestTable = physicalTables.stream().reduce(betterTable).get(); + PhysicalTable bestTable = physicalTables.stream() + .reduce(getBetterTableOperator(requestConstraints)).get(); REGISTRY.meter("request.physical.table." + bestTable.getName() + "." + bestTable.getTimeGrain()).mark(); LOG.trace("Found best Table: {}", bestTable); return bestTable; } catch (NoMatchFoundException me) { // Blow up if we couldn't match a table, log and return if we can - logMatchException(apiRequest, minimumTableTimeGrain, innerQuery); + LOG.error(ErrorMessageFormat.NO_PHYSICAL_TABLE_MATCHED.logFormat( + requestConstraints.getAllDimensionNames(), + requestConstraints.getLogicalMetricNames(), + requestConstraints.getMinimumGranularity() + )); throw me; } } - - /** - * Log out inability to find a matching table. - * - * @param apiRequest Request for which we're trying to find a table - * @param minimumTableTimeGrain Minimum grain that we needed to meet - * @param innerQuery Innermost query for the query we were trying to match - */ - public void logMatchException( - DataApiRequest apiRequest, - Granularity minimumTableTimeGrain, - TemplateDruidQuery innerQuery - ) { - // Get the dimensions and metrics as lists of names - Set requestDimensionNames = TableUtils.getDimensions(apiRequest, innerQuery) - .map(Dimension::getApiName) - .collect(Collectors.toSet()); - - Set requestMetricNames = apiRequest.getLogicalMetrics().stream() - .map(LogicalMetric::getName) - .collect(Collectors.toCollection(LinkedHashSet::new)); - - String msg = ErrorMessageFormat.NO_PHYSICAL_TABLE_MATCHED.logFormat( - requestDimensionNames, - requestMetricNames, - minimumTableTimeGrain - ); - LOG.error(msg); - } } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/DataSourceConstraint.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/DataSourceConstraint.java new file mode 100644 index 0000000000..b7f0eb4020 --- /dev/null +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/DataSourceConstraint.java @@ -0,0 +1,88 @@ +// Copyright 2017 Yahoo Inc. +// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. +package com.yahoo.bard.webservice.table.resolver; + +import com.yahoo.bard.webservice.data.dimension.Dimension; +import com.yahoo.bard.webservice.druid.model.query.DruidAggregationQuery; +import com.yahoo.bard.webservice.web.ApiFilter; +import com.yahoo.bard.webservice.web.DataApiRequest; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Constraints for retrieving potential table availability for a given query. + */ +public class DataSourceConstraint { + + private final Set requestDimensions; + private final Set filterDimensions; + private final Set metricDimensions; + private final Set metricNames; + private final Map> apiFilters; + private final Set allDimensions; + private final Set allDimensionNames; + private final Set allColumnNames; + + /** + * Constructor. + * + * @param dataApiRequest Api request containing the constraints information. + * @param templateDruidQuery Query containing metric constraint information. + */ + public DataSourceConstraint(DataApiRequest dataApiRequest, DruidAggregationQuery templateDruidQuery) { + this.requestDimensions = Collections.unmodifiableSet(dataApiRequest.getDimensions()); + this.filterDimensions = Collections.unmodifiableSet(dataApiRequest.getFilterDimensions()); + this.metricDimensions = Collections.unmodifiableSet(templateDruidQuery.getMetricDimensions()); + this.metricNames = Collections.unmodifiableSet(templateDruidQuery.getDependentFieldNames()); + this.apiFilters = Collections.unmodifiableMap(dataApiRequest.getFilters()); + this.allDimensions = Collections.unmodifiableSet(Stream.of( + getRequestDimensions().stream(), + getFilterDimensions().stream(), + getMetricDimensions().stream() + ).flatMap(Function.identity()).collect(Collectors.toSet())); + this.allDimensionNames = Collections.unmodifiableSet(allDimensions.stream() + .map(Dimension::getApiName) + .collect(Collectors.toSet())); + this.allColumnNames = Collections.unmodifiableSet(Stream.concat( + allDimensionNames.stream(), + metricNames.stream() + ).collect(Collectors.toSet())); + } + + public Set getRequestDimensions() { + return requestDimensions; + } + + public Set getFilterDimensions() { + return filterDimensions; + } + + public Set getMetricDimensions() { + return metricDimensions; + } + + public Set getMetricNames() { + return metricNames; + } + + public Set getAllDimensions() { + return allDimensions; + } + + public Set getAllDimensionNames() { + return allDimensionNames; + } + + public Set getAllColumnNames() { + return allColumnNames; + } + + public Map> getApiFilters() { + return apiFilters; + } +} diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/DefaultPhysicalTableResolver.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/DefaultPhysicalTableResolver.java index 16a9599f15..a5cba17c53 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/DefaultPhysicalTableResolver.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/DefaultPhysicalTableResolver.java @@ -1,14 +1,12 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver; import com.yahoo.bard.webservice.config.BardFeatureFlag; import com.yahoo.bard.webservice.data.PartialDataHandler; -import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery; import com.yahoo.bard.webservice.data.volatility.VolatileIntervalsService; import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.util.ChainingComparator; -import com.yahoo.bard.webservice.web.DataApiRequest; import java.util.ArrayList; import java.util.Arrays; @@ -51,30 +49,27 @@ public DefaultPhysicalTableResolver( } @Override - public List getMatchers(DataApiRequest apiRequest, TemplateDruidQuery query) { - SchemaPhysicalTableMatcher schemaMatcher = new SchemaPhysicalTableMatcher( - apiRequest, - query, - resolveAcceptingGrain.apply(apiRequest, query) + public List getMatchers(QueryPlanningConstraint requestConstraints) { + return Arrays.asList( + new SchemaPhysicalTableMatcher(requestConstraints), + new AggregatableDimensionsMatcher(requestConstraints), + new TimeAlignmentPhysicalTableMatcher(requestConstraints) ); - TimeAlignmentPhysicalTableMatcher timeAlignmentMatcher = new TimeAlignmentPhysicalTableMatcher(apiRequest); - AggregatableDimensionsMatcher aggregatabilityMatcher = new AggregatableDimensionsMatcher(apiRequest, query); - - return Arrays.asList(schemaMatcher, aggregatabilityMatcher, timeAlignmentMatcher); } @Override - public BinaryOperator getBetterTableOperator(DataApiRequest apiRequest, TemplateDruidQuery query) { + public BinaryOperator getBetterTableOperator(QueryPlanningConstraint requestConstraints) { List> comparators = new ArrayList<>(); if (BardFeatureFlag.PARTIAL_DATA.isOn()) { - comparators.add(new PartialTimeComparator(apiRequest, query, partialDataHandler)); comparators.add( - new VolatileTimeComparator(apiRequest, query, partialDataHandler, volatileIntervalsService) - ); + new PartialTimeComparator(requestConstraints, partialDataHandler)); + comparators.add( + new VolatileTimeComparator(requestConstraints, partialDataHandler, volatileIntervalsService)); } comparators.add(COMPARE_GRANULARITY); comparators.add(CARDINALITY_COMPARATOR); + ChainingComparator tableComparator = new ChainingComparator<>(comparators); return BinaryOperator.minBy(tableComparator); } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/PartialTimeComparator.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/PartialTimeComparator.java index 7b1e1c9127..95faca5ee0 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/PartialTimeComparator.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/PartialTimeComparator.java @@ -1,13 +1,11 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver; import com.yahoo.bard.webservice.data.PartialDataHandler; -import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery; import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.util.IntervalUtils; import com.yahoo.bard.webservice.util.SimplifiedIntervalList; -import com.yahoo.bard.webservice.web.DataApiRequest; import java.util.Collections; import java.util.Comparator; @@ -18,19 +16,16 @@ public class PartialTimeComparator implements Comparator { private final PartialDataHandler partialDataHandler; - private final DataApiRequest request; - private final TemplateDruidQuery query; + private final QueryPlanningConstraint requestConstraints; /** * Constructor. * - * @param request Request for which we're comparing parial time - * @param query TDQ for which time is being compared + * @param requestConstraints contains the request constraints extracted from DataApiRequest and TemplateDruidQuery * @param handler Handler for Partial Data */ - public PartialTimeComparator(DataApiRequest request, TemplateDruidQuery query, PartialDataHandler handler) { - this.request = request; - this.query = query; + public PartialTimeComparator(QueryPlanningConstraint requestConstraints, PartialDataHandler handler) { + this.requestConstraints = requestConstraints; this.partialDataHandler = handler; } @@ -47,20 +42,18 @@ public int compare(PhysicalTable left, PhysicalTable right) { // choose table with most data available for given columns long missingDurationLeft = IntervalUtils.getTotalDuration( partialDataHandler.findMissingTimeGrainIntervals( - request, - query, + requestConstraints.getAllColumnNames(), Collections.singleton(left), - new SimplifiedIntervalList(request.getIntervals()), - request.getGranularity() + new SimplifiedIntervalList(requestConstraints.getIntervals()), + requestConstraints.getRequestGranularity() ) ); long missingDurationRight = IntervalUtils.getTotalDuration( partialDataHandler.findMissingTimeGrainIntervals( - request, - query, + requestConstraints.getAllColumnNames(), Collections.singleton(right), - new SimplifiedIntervalList(request.getIntervals()), - request.getGranularity() + new SimplifiedIntervalList(requestConstraints.getIntervals()), + requestConstraints.getRequestGranularity() ) ); long difference = missingDurationLeft - missingDurationRight; diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/PhysicalTableResolver.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/PhysicalTableResolver.java index e2b90bb5a7..6faf7181d2 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/PhysicalTableResolver.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/PhysicalTableResolver.java @@ -1,10 +1,8 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver; -import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery; import com.yahoo.bard.webservice.table.PhysicalTable; -import com.yahoo.bard.webservice.web.DataApiRequest; import java.util.Collection; @@ -18,8 +16,7 @@ public interface PhysicalTableResolver { * Choose the best fit Physical Table from a table group. * * @param candidateTables The tables being considered for match - * @param apiRequest The ApiRequest for the query - * @param query a partial query representation + * @param requestConstraints Contains the request constraints extracted from DataApiRequest and TemplateDruidQuery * * @return The table, if any, that satisfies all criteria and best matches the query * @@ -27,7 +24,6 @@ public interface PhysicalTableResolver { */ PhysicalTable resolve( Collection candidateTables, - DataApiRequest apiRequest, - TemplateDruidQuery query + QueryPlanningConstraint requestConstraints ) throws NoMatchFoundException; } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/QueryPlanningConstraint.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/QueryPlanningConstraint.java new file mode 100644 index 0000000000..88f0259dcd --- /dev/null +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/QueryPlanningConstraint.java @@ -0,0 +1,70 @@ +// Copyright 2017 Yahoo Inc. +// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. +package com.yahoo.bard.webservice.table.resolver; + +import com.yahoo.bard.webservice.data.metric.LogicalMetric; +import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery; +import com.yahoo.bard.webservice.druid.model.query.Granularity; +import com.yahoo.bard.webservice.table.LogicalTable; +import com.yahoo.bard.webservice.web.DataApiRequest; + +import org.joda.time.Interval; + +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Constraints used to match and resolve the best table for a given query. + */ +public class QueryPlanningConstraint extends DataSourceConstraint { + + private final LogicalTable logicalTable; + private final Set intervals; + private final Set logicalMetrics; + private final Granularity minimumGranularity; + private final Granularity requestGranularity; + private final Set logicalMetricNames; + + /** + * Constructor. + * + * @param dataApiRequest Api request containing the constraints information. + * @param templateDruidQuery Query containing metric constraint information. + */ + public QueryPlanningConstraint(DataApiRequest dataApiRequest, TemplateDruidQuery templateDruidQuery) { + super(dataApiRequest, templateDruidQuery); + + this.logicalTable = dataApiRequest.getTable(); + this.intervals = Collections.unmodifiableSet(dataApiRequest.getIntervals()); + this.logicalMetrics = Collections.unmodifiableSet(dataApiRequest.getLogicalMetrics()); + this.minimumGranularity = new RequestQueryGranularityResolver().apply(dataApiRequest, templateDruidQuery); + this.requestGranularity = dataApiRequest.getGranularity(); + this.logicalMetricNames = Collections.unmodifiableSet( + logicalMetrics.stream().map(LogicalMetric::getName).collect(Collectors.toSet())); + } + + public LogicalTable getLogicalTable() { + return logicalTable; + } + + public Set getIntervals() { + return intervals; + } + + public Set getLogicalMetrics() { + return logicalMetrics; + } + + public Set getLogicalMetricNames() { + return logicalMetricNames; + } + + public Granularity getMinimumGranularity() { + return minimumGranularity; + } + + public Granularity getRequestGranularity() { + return requestGranularity; + } +} diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/SchemaPhysicalTableMatcher.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/SchemaPhysicalTableMatcher.java index f8f42796bf..f50ee10c0c 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/SchemaPhysicalTableMatcher.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/SchemaPhysicalTableMatcher.java @@ -1,17 +1,11 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver; import static com.yahoo.bard.webservice.web.ErrorMessageFormat.TABLE_SCHEMA_UNDEFINED; -import com.yahoo.bard.webservice.data.dimension.Dimension; -import com.yahoo.bard.webservice.data.metric.LogicalMetric; -import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery; -import com.yahoo.bard.webservice.druid.model.query.Granularity; import com.yahoo.bard.webservice.table.Column; import com.yahoo.bard.webservice.table.PhysicalTable; -import com.yahoo.bard.webservice.util.TableUtils; -import com.yahoo.bard.webservice.web.DataApiRequest; import com.yahoo.bard.webservice.web.ErrorMessageFormat; import org.slf4j.Logger; @@ -30,26 +24,20 @@ public class SchemaPhysicalTableMatcher implements PhysicalTableMatcher { public static final ErrorMessageFormat MESSAGE_FORMAT = TABLE_SCHEMA_UNDEFINED; - private final DataApiRequest request; - private final Granularity granularity; - private final TemplateDruidQuery query; + private final QueryPlanningConstraint requestConstraints; /** * Constructor saves metrics, dimensions, coarsest time grain, and logical table name (for logging). * - * @param request The request whose dimensions are being matched on - * @param query The query whose columns are being matched - * @param granularity The granularity that tables under test must satisfy + * @param requestConstraints contains the request constraints extracted from DataApiRequest and TemplateDruidQuery */ - public SchemaPhysicalTableMatcher(DataApiRequest request, TemplateDruidQuery query, Granularity granularity) { - this.request = request; - this.granularity = granularity; - this.query = query; + public SchemaPhysicalTableMatcher(QueryPlanningConstraint requestConstraints) { + this.requestConstraints = requestConstraints; } @Override public boolean test(PhysicalTable table) { - if (!granularity.satisfiedBy(table.getSchema().getTimeGrain())) { + if (!requestConstraints.getMinimumGranularity().satisfiedBy(table.getSchema().getTimeGrain())) { return false; } @@ -57,24 +45,22 @@ public boolean test(PhysicalTable table) { .map(Column::getName) .collect(Collectors.toCollection(LinkedHashSet::new)); - Set columnNames = TableUtils.getColumnNames(request, query.getInnermostQuery()); + Set columnNames = requestConstraints.getAllColumnNames(); return supplyNames.containsAll(columnNames); } @Override public NoMatchFoundException noneFoundException() { - String logicalTableName = request.getTable().getName(); - Set logicalMetrics = request.getLogicalMetrics().stream() - .map(LogicalMetric::getName) - .collect(Collectors.toSet()); - Set dimensions = request.getDimensions().stream() - .map(Dimension::getApiName) - .collect(Collectors.toSet()); - - LOG.error(MESSAGE_FORMAT.logFormat(logicalTableName, dimensions, logicalMetrics, granularity.getName())); + String logicalTableName = requestConstraints.getLogicalTable().getName(); + Set logicalMetrics = requestConstraints.getLogicalMetricNames(); + Set dimensions = requestConstraints.getAllDimensionNames(); + String grainName = requestConstraints.getMinimumGranularity().getName(); + LOG.error( + MESSAGE_FORMAT.logFormat(logicalTableName, dimensions, logicalMetrics, grainName) + ); return new NoMatchFoundException( - MESSAGE_FORMAT.format(logicalTableName, dimensions, logicalMetrics, granularity.getName()) + MESSAGE_FORMAT.format(logicalTableName, dimensions, logicalMetrics, grainName) ); } } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/TimeAlignmentPhysicalTableMatcher.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/TimeAlignmentPhysicalTableMatcher.java index 0b18b37e9f..997528eb8f 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/TimeAlignmentPhysicalTableMatcher.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/TimeAlignmentPhysicalTableMatcher.java @@ -1,11 +1,10 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver; import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.table.availability.IsTableAligned; import com.yahoo.bard.webservice.table.availability.IsTableStartAlignedWithIntervals; -import com.yahoo.bard.webservice.web.DataApiRequest; import com.yahoo.bard.webservice.web.ErrorMessageFormat; import org.joda.time.Interval; @@ -31,15 +30,15 @@ public class TimeAlignmentPhysicalTableMatcher implements PhysicalTableMatcher { * Stores the request table name and intervals and creates a predicate to test a physical table based on request * intervals. * - * @param request The Api Request being matched against + * @param requestConstraints Contains the request constraints extracted from DataApiRequest and TemplateDruidQuery */ - public TimeAlignmentPhysicalTableMatcher(DataApiRequest request) { - if (request.getIntervals().isEmpty()) { + public TimeAlignmentPhysicalTableMatcher(QueryPlanningConstraint requestConstraints) { + if (requestConstraints.getIntervals().isEmpty()) { throw new IllegalStateException("Intervals cannot be empty"); } - logicalTableName = request.getTable().getName(); - requestIntervals = request.getIntervals(); - isTableAligned = new IsTableStartAlignedWithIntervals(request.getIntervals()); + logicalTableName = requestConstraints.getLogicalTable().getName(); + requestIntervals = requestConstraints.getIntervals(); + isTableAligned = new IsTableStartAlignedWithIntervals(requestConstraints.getIntervals()); } @Override diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/VolatileTimeComparator.java b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/VolatileTimeComparator.java index e957d7b7db..690ef2b49e 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/VolatileTimeComparator.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/table/resolver/VolatileTimeComparator.java @@ -1,16 +1,13 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver; -import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.data.PartialDataHandler; import com.yahoo.bard.webservice.data.volatility.VolatileIntervalsService; -import com.yahoo.bard.webservice.druid.model.query.DruidAggregationQuery; import com.yahoo.bard.webservice.druid.model.query.Granularity; +import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.util.IntervalUtils; import com.yahoo.bard.webservice.util.SimplifiedIntervalList; -import com.yahoo.bard.webservice.util.TableUtils; -import com.yahoo.bard.webservice.web.DataApiRequest; import java.util.Collections; import java.util.Comparator; @@ -20,28 +17,24 @@ */ public class VolatileTimeComparator implements Comparator { - private final DataApiRequest apiRequest; - private final DruidAggregationQuery query; + private final QueryPlanningConstraint requestConstraints; private final PartialDataHandler partialDataHandler; private final VolatileIntervalsService volatileIntervalsService; /** * Builds a table comparator that compares tables based on how much data there is in their volatile intervals. * - * @param query The query to send to druid, and that requires a table - * @param apiRequest The data request + * @param requestConstraints Contains the request constraints extracted from DataApiRequest and TemplateDruidQuery * @param partialDataHandler A service for computing partial data information * @param volatileIntervalsService A service to extract the intervals in a query that are volatile with respect * to a given table */ public VolatileTimeComparator( - DataApiRequest apiRequest, - DruidAggregationQuery query, + QueryPlanningConstraint requestConstraints, PartialDataHandler partialDataHandler, VolatileIntervalsService volatileIntervalsService ) { - this.apiRequest = apiRequest; - this.query = query; + this.requestConstraints = requestConstraints; this.partialDataHandler = partialDataHandler; this.volatileIntervalsService = volatileIntervalsService; } @@ -87,12 +80,11 @@ public int compare(PhysicalTable left, PhysicalTable right) { * request granularity, and present at the table granularity */ private long getAvailableVolatileDataDuration(PhysicalTable table) { - SimplifiedIntervalList requestIntervals = new SimplifiedIntervalList(apiRequest.getIntervals()); - Granularity apiRequestGranularity = apiRequest.getGranularity(); + SimplifiedIntervalList requestIntervals = new SimplifiedIntervalList(requestConstraints.getIntervals()); + Granularity apiRequestGranularity = requestConstraints.getRequestGranularity(); // First, find the volatile intervals that are also partial at the request grain. SimplifiedIntervalList volatilePartialRequestIntervals = partialDataHandler.findMissingTimeGrainIntervals( - apiRequest, - query, + requestConstraints.getAllColumnNames(), Collections.singleton(table), volatileIntervalsService.getVolatileIntervals(apiRequestGranularity, requestIntervals, table), apiRequestGranularity @@ -100,7 +92,7 @@ private long getAvailableVolatileDataDuration(PhysicalTable table) { //Next find the intervals on the physical table that are available. SimplifiedIntervalList tableAvailability = partialDataHandler.getAvailability( table, - TableUtils.getColumnNames(apiRequest, query) + requestConstraints.getAllColumnNames() ); //Take the duration of their intersection. return IntervalUtils.getTotalDuration(tableAvailability.intersect(volatilePartialRequestIntervals)); diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/handlers/PartialDataRequestHandler.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/handlers/PartialDataRequestHandler.java index ebdbd7c65e..62a9dd1430 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/handlers/PartialDataRequestHandler.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/handlers/PartialDataRequestHandler.java @@ -1,4 +1,4 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.web.handlers; @@ -11,6 +11,7 @@ import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.table.PhysicalTableDictionary; import com.yahoo.bard.webservice.util.SimplifiedIntervalList; +import com.yahoo.bard.webservice.util.TableUtils; import com.yahoo.bard.webservice.web.DataApiRequest; import com.yahoo.bard.webservice.web.responseprocessors.MappingResponseProcessor; import com.yahoo.bard.webservice.web.responseprocessors.ResponseContext; @@ -73,8 +74,7 @@ public boolean handleRequest( // Gather the missing intervals SimplifiedIntervalList missingIntervals = partialDataHandler.findMissingTimeGrainIntervals( - request, - druidQuery, + TableUtils.getColumnNames(request, druidQuery), physicalTables, new SimplifiedIntervalList(request.getIntervals()), request.getGranularity() diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/PartialDataHandlerSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/PartialDataHandlerSpec.groovy index ec171eb6b2..fe0b39743f 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/PartialDataHandlerSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/PartialDataHandlerSpec.groovy @@ -10,16 +10,13 @@ import static org.joda.time.DateTimeZone.UTC import com.yahoo.bard.webservice.data.dimension.Dimension import com.yahoo.bard.webservice.data.dimension.DimensionDictionary import com.yahoo.bard.webservice.data.dimension.impl.KeyValueStoreDimension -import com.yahoo.bard.webservice.druid.model.datasource.QueryDataSource -import com.yahoo.bard.webservice.druid.model.datasource.TableDataSource import com.yahoo.bard.webservice.druid.model.query.AllGranularity -import com.yahoo.bard.webservice.druid.model.query.Granularity import com.yahoo.bard.webservice.druid.model.query.GroupByQuery import com.yahoo.bard.webservice.metadata.SegmentMetadata import com.yahoo.bard.webservice.table.ConcretePhysicalTable import com.yahoo.bard.webservice.table.PhysicalTable +import com.yahoo.bard.webservice.table.resolver.QueryPlanningConstraint import com.yahoo.bard.webservice.util.SimplifiedIntervalList -import com.yahoo.bard.webservice.web.DataApiRequest import org.joda.time.Interval @@ -41,13 +38,11 @@ class PartialDataHandlerSpec extends Specification { [] as Set, ["userDeviceType": "user_device_type"] )] as Set - Set columnNames + Set columnNames GroupByQuery groupByQuery = Mock(GroupByQuery.class) - DimensionDictionary dimensionDictionary - - DataApiRequest apiRequest + QueryPlanningConstraint dataSourceConstraint def cleanup() { PERMISSIVE_COLUMN_AVAILABILITY.setOn(originalConfig) @@ -67,30 +62,8 @@ class PartialDataHandlerSpec extends Specification { dimensionDictionary = new DimensionDictionary([dim1, dim2, dim3] as Set) // Setup mock Request - apiRequest = Mock(DataApiRequest.class) - apiRequest.getDimensions() >> { [dim1, dim2] as Collection } - apiRequest.getFilterDimensions() >> { [] as Set } - apiRequest.getFilters() >> { [:] } - - // setup mock inner query - GroupByQuery innerQuery = Mock(GroupByQuery.class) - innerQuery.getDataSource() >> { - new TableDataSource( - new ConcretePhysicalTable( - "basefact_network", DAY.buildZonedTimeGrain(UTC), - [] as Set, - ["userDeviceType": "user_device_type"] - ) - ) - } - innerQuery.getDependentFieldNames() >> { ["page_views"] as Set } - - groupByQuery.getGranularity() >> WEEK - groupByQuery.getIntervals() >> { [new Interval("2014-06-30/2014-07-28")] as List } - groupByQuery.getDataSource() >> { new QueryDataSource(innerQuery) } - groupByQuery.getDependentFieldNames() >> { innerQuery.getDependentFieldNames() } - groupByQuery.getInnermostQuery() >> { innerQuery } - groupByQuery.getMetricDimensions() >> [] + dataSourceConstraint = Stub(QueryPlanningConstraint) + dataSourceConstraint.getAllColumnNames() >> columnNames /* * dim1 is missing four days of data internally, dim2 and dim3 are complete over the period and page_views is @@ -106,15 +79,6 @@ class PartialDataHandlerSpec extends Specification { tables*.resetColumns(segments, dimensionDictionary) } - def setupApiRequest( - Granularity requestGranularity = WEEK, - Collection requestedIntervals = [new Interval("2014-06-30/2014-07-28")] - ) { - apiRequest.getIntervals() >> { requestedIntervals } - apiRequest.getGranularity() >> requestGranularity - apiRequest - } - @Unroll def "Merging all columns produces #expected with #comment"() { setup: @@ -149,15 +113,15 @@ class PartialDataHandlerSpec extends Specification { setup: PERMISSIVE_COLUMN_AVAILABILITY.setOn(false) SimplifiedIntervalList expectedIntervals = buildIntervals(["2014-06-30/2014-07-14"]) - apiRequest = setupApiRequest() + dataSourceConstraint.getRequestGranularity() >> WEEK + dataSourceConstraint.getIntervals() >> [new Interval("2014-06-30/2014-07-28")] expect: expectedIntervals == partialDataHandler.findMissingTimeGrainIntervals( - apiRequest, - groupByQuery, + dataSourceConstraint.getAllColumnNames(), tables, - new SimplifiedIntervalList(apiRequest.intervals), - apiRequest.granularity + new SimplifiedIntervalList(dataSourceConstraint.getIntervals()), + dataSourceConstraint.getRequestGranularity() ) } @@ -165,15 +129,15 @@ class PartialDataHandlerSpec extends Specification { setup: PERMISSIVE_COLUMN_AVAILABILITY.setOn(false) SimplifiedIntervalList requestedIntervals = buildIntervals(["2014-06-30/2014-07-14"]) - apiRequest = setupApiRequest(AllGranularity.INSTANCE, requestedIntervals) + dataSourceConstraint.getRequestGranularity() >> AllGranularity.INSTANCE + dataSourceConstraint.getIntervals() >> requestedIntervals expect: requestedIntervals == partialDataHandler.findMissingTimeGrainIntervals( - apiRequest, - groupByQuery, + dataSourceConstraint.getAllColumnNames(), tables, - new SimplifiedIntervalList(apiRequest.intervals), - apiRequest.granularity + new SimplifiedIntervalList(dataSourceConstraint.getIntervals()), + dataSourceConstraint.getRequestGranularity() ) } diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToDefaultDimensionSpecSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToDefaultDimensionSpecSpec.groovy index 5a78a22102..50f97289f1 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToDefaultDimensionSpecSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/DimensionToDefaultDimensionSpecSpec.groovy @@ -51,6 +51,7 @@ class DimensionToDefaultDimensionSpecSpec extends Specification { apiRequest.getTopN() >> OptionalInt.empty() apiRequest.getSorts() >> ([]) apiRequest.getCount() >> OptionalInt.empty() + apiRequest.getFilters() >> Collections.emptyMap() } def "Serialize to apiName when apiName and physicalName of a dimension is the same"() { diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/LookupDimensionToDimensionSpecSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/LookupDimensionToDimensionSpecSpec.groovy index 6d982d23f0..a2188fa63f 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/LookupDimensionToDimensionSpecSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/LookupDimensionToDimensionSpecSpec.groovy @@ -52,6 +52,7 @@ class LookupDimensionToDimensionSpecSpec extends Specification{ apiRequest.getTopN() >> OptionalInt.empty() apiRequest.getSorts() >> ([]) apiRequest.getCount() >> OptionalInt.empty() + apiRequest.getFilters() >> Collections.emptyMap() } def "Given lookup dimension with no namespace serialize using dimension serializer"() { diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/RegisteredLookupDimensionToDimensionSpecSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/RegisteredLookupDimensionToDimensionSpecSpec.groovy index d3f7dc1dcc..54222a18cd 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/RegisteredLookupDimensionToDimensionSpecSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/druid/serializers/RegisteredLookupDimensionToDimensionSpecSpec.groovy @@ -52,6 +52,7 @@ class RegisteredLookupDimensionToDimensionSpecSpec extends Specification{ apiRequest.getTopN() >> OptionalInt.empty() apiRequest.getSorts() >> ([]) apiRequest.getCount() >> OptionalInt.empty() + apiRequest.getFilters() >> Collections.emptyMap() } def "Given registered lookup dimension with no lookup serialize using dimension serializer"() { diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/BasePhysicalTableResolverSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/BasePhysicalTableResolverSpec.groovy index 2d75dc9b44..c67b88f336 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/BasePhysicalTableResolverSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/BasePhysicalTableResolverSpec.groovy @@ -1,72 +1,44 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver -import static com.yahoo.bard.webservice.data.time.DefaultTimeGrain.DAY -import static org.joda.time.DateTimeZone.UTC - -import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery -import com.yahoo.bard.webservice.druid.model.query.AllGranularity -import com.yahoo.bard.webservice.table.ConcretePhysicalTable -import com.yahoo.bard.webservice.table.LogicalTable import com.yahoo.bard.webservice.table.PhysicalTable -import com.yahoo.bard.webservice.table.TableGroup -import com.yahoo.bard.webservice.web.DataApiRequest - -import com.google.common.collect.Sets import spock.lang.Shared import spock.lang.Specification import spock.lang.Unroll import java.util.function.BinaryOperator + /** * Tests for methods in BasePhysicalResolverSpec */ class BasePhysicalTableResolverSpec extends Specification { - List tables + @Shared PhysicalTable one, two, three @Shared PhysicalTableMatcher matchAllButTablesNamedOne, matchThree, matchAll - @Shared BinaryOperator pickFirst - @Shared BinaryOperator pickLast - @Shared PhysicalTable one = Mock(PhysicalTable) - @Shared PhysicalTable two = Mock(PhysicalTable) - @Shared PhysicalTable three = Mock(PhysicalTable) - - @Shared NoMatchFoundException noMatchNotAny = new NoMatchFoundException("notAny") - @Shared NoMatchFoundException noMatchNotOne = new NoMatchFoundException("notOne") - @Shared NoMatchFoundException noMatchThree = new NoMatchFoundException("three") - - DataApiRequest request = Mock(DataApiRequest) - TemplateDruidQuery query = Mock(TemplateDruidQuery) - - BasePhysicalTableResolver physicalTableResolver = new BasePhysicalTableResolver() { - - List matchers - BinaryOperator betterTable - - @Override - List getMatchers(DataApiRequest apiRequest, TemplateDruidQuery query) { - return matchers - } - - @Override - BinaryOperator getBetterTableOperator(DataApiRequest apiRequest, TemplateDruidQuery query) { - return betterTable - } - } + @Shared BinaryOperator pickFirst, pickLast + @Shared NoMatchFoundException noMatchNotAny, noMatchNotOne, noMatchThree + BasePhysicalTableResolver physicalTableResolver + QueryPlanningConstraint dataSourceConstraint def setupSpec() { + one = Mock(PhysicalTable) + two = Mock(PhysicalTable) + three = Mock(PhysicalTable) + one.getName() >> "one" - one.toString() >> "one" two.getName() >> "two" - two.toString() >> "two" three.getName() >> "three" - three.toString() >> "three" pickFirst = { PhysicalTable table1, PhysicalTable table2 -> table1 } as BinaryOperator pickLast = { PhysicalTable table1, PhysicalTable table2 -> table2 } as BinaryOperator + + noMatchNotAny = new NoMatchFoundException("No Match Found: notAny") + noMatchNotOne = new NoMatchFoundException("No Match Found: notOne") + noMatchThree = new NoMatchFoundException("No Match Found: three") + matchAll = new PhysicalTableMatcher() { @Override boolean test(PhysicalTable table) { @@ -120,28 +92,32 @@ class BasePhysicalTableResolverSpec extends Specification { } def setup() { - request.getGranularity() >> AllGranularity.INSTANCE - query.getInnermostQuery() >> query - query.getDimensions() >> [] - query.getMetricDimensions() >> ([] as Set) - query.getDependentFieldNames() >> ([] as Set) - request.getFilterDimensions() >> [] - request.getDimensions() >> ([] as Set) - - LogicalTable logical = Mock(LogicalTable.class) - TableGroup group = Mock(TableGroup.class) - logical.getTableGroup() >> group - PhysicalTable table = new ConcretePhysicalTable("table_name", DAY.buildZonedTimeGrain(UTC), [] as Set, [:]) - group.getPhysicalTables() >> Sets.newHashSet(table) - request.getTable() >> logical + dataSourceConstraint = Mock(QueryPlanningConstraint) + + physicalTableResolver = new BasePhysicalTableResolver() { + + List matchers + BinaryOperator betterTable + + @Override + List getMatchers(QueryPlanningConstraint requestConstraints) { + return matchers + } + + @Override + BinaryOperator getBetterTableOperator(QueryPlanningConstraint requestConstraints) { + return betterTable + } + } } + @Unroll def "Test matchers with no empties to #expected"() { setup: - physicalTableResolver.matchers = matchers as List + physicalTableResolver.matchers = matchers expect: - physicalTableResolver.filter( [one, two, three] as List, request, query) == expected as Set + physicalTableResolver.filter([one, two, three], dataSourceConstraint) == expected.toSet() where: matchers | expected @@ -154,13 +130,15 @@ class BasePhysicalTableResolverSpec extends Specification { @Unroll def "Test matchers with throw no match exception with #matchers and #supply"() { setup: - physicalTableResolver.matchers = matchers as List + physicalTableResolver.matchers = matchers when: - physicalTableResolver.filter( supply as List, request, query) + physicalTableResolver.filter( supply, dataSourceConstraint) then: - thrown(NoMatchFoundException) + NoMatchFoundException noMatchFoundException = thrown() + noMatchFoundException.message.startsWith('No Match Found: ') + where: matchers | supply | noMatch @@ -178,7 +156,7 @@ class BasePhysicalTableResolverSpec extends Specification { physicalTableResolver.betterTable = better expect: - physicalTableResolver.resolve([one, two, three], request, query) == expected + physicalTableResolver.resolve([one, two, three], dataSourceConstraint) == expected where: matchers | better | expected diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/DefaultPhysicalTableResolverSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/DefaultPhysicalTableResolverSpec.groovy index 183a73328c..322e6e0c7c 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/DefaultPhysicalTableResolverSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/DefaultPhysicalTableResolverSpec.groovy @@ -1,4 +1,4 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver @@ -138,6 +138,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { apiRequest.granularity >> granularity apiRequest.filterDimensions >> prototype['filterDimensions'] apiRequest.logicalMetrics >> prototype['logicalMetrics'] + apiRequest.getFilters() >> Collections.emptyMap() return apiRequest } @@ -161,7 +162,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.filter([table], apiRequest, query) == [table] as Set + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) == [table] as Set where: grain | dimensions | table @@ -193,7 +194,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.filter([table], apiRequest, query) == [table] as Set + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) == [table] as Set where: grainName | dimensions | table @@ -224,7 +225,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) when: - resolver.filter([table], apiRequest, query) + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) then: thrown(NoMatchFoundException) @@ -251,7 +252,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.filter([table], apiRequest, query) == [table] as Set + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) == [table] as Set where: dimensions | table @@ -275,7 +276,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) when: - resolver.filter([table], apiRequest, query) + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) then: thrown(NoMatchFoundException) @@ -300,7 +301,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.filter([table], apiRequest, query) == [table] as Set + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) == [table] as Set where: grain | metrics | table @@ -326,7 +327,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) when: - resolver.filter([table], apiRequest, query) + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) then: thrown(NoMatchFoundException) @@ -351,7 +352,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.filter([table], apiRequest, query) == [table] as Set + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) == [table] as Set where: grain | metrics | table @@ -378,7 +379,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) when: - resolver.filter([table], apiRequest, query) + resolver.filter([table], new QueryPlanningConstraint(apiRequest, query)) then: thrown(NoMatchFoundException) @@ -401,7 +402,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { apiRequestPrototype['logicalMetrics'] = metricsForNameSet(queryPrototype['dependantFieldNames'] as Set) DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) - BinaryOperator betterTable = resolver.getBetterTableOperator(apiRequest, query) + BinaryOperator betterTable = resolver.getBetterTableOperator(new QueryPlanningConstraint(apiRequest, query)) expect: [table1, table2].stream().reduce(betterTable).get() == expected @@ -427,7 +428,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { apiRequestPrototype['logicalMetrics'] = metricsForNameSet(queryPrototype['dependantFieldNames'] as Set) DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) - BinaryOperator betterTable = resolver.getBetterTableOperator(apiRequest, query) + BinaryOperator betterTable = resolver.getBetterTableOperator(new QueryPlanningConstraint(apiRequest, query)) expect: [(PhysicalTable) table1, (PhysicalTable) table2].stream().reduce(betterTable).get() == [table1, table1, table2].get(which) @@ -482,7 +483,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.resolve(resources.tg1All.physicalTables, apiRequest, query) == expected + resolver.resolve(resources.tg1All.physicalTables, new QueryPlanningConstraint(apiRequest, query)) == expected where: intervals | grain || expected @@ -512,7 +513,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.resolve(resources.tg1All.physicalTables, apiRequest, query) == expected + resolver.resolve(resources.tg1All.physicalTables, new QueryPlanningConstraint(apiRequest, query)) == expected where: interval | grain || expected @@ -551,7 +552,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: "The table with more data available is preferred" - localResolver.resolve([resources.volatileHourTable, resources.volatileDayTable], apiRequest, query) == expected + localResolver.resolve([resources.volatileHourTable, resources.volatileDayTable], new QueryPlanningConstraint(apiRequest, query)) == expected where: hourAvailable | hourVolatile | dayAvailable | dayVolatile | grain || expected @@ -579,7 +580,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.resolve(tablegroup.physicalTables, apiRequest, query) == expected + resolver.resolve(tablegroup.physicalTables, new QueryPlanningConstraint(apiRequest, query)) == expected where: grain | dimensions | tablegroup | metrics | expected @@ -607,7 +608,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) when: - resolver.resolve(tablegroup.physicalTables, apiRequest, query) + resolver.resolve(tablegroup.physicalTables, new QueryPlanningConstraint(apiRequest, query)) then: thrown(NoMatchFoundException) @@ -632,7 +633,7 @@ class DefaultPhysicalTableResolverSpec extends Specification { DataApiRequest apiRequest = buildDataApiRequest(apiRequestPrototype) expect: - resolver.resolve(tablegroup.physicalTables, apiRequest, query) == table + resolver.resolve(tablegroup.physicalTables, new QueryPlanningConstraint(apiRequest, query)) == table where: grain | dimensions | tablegroup | table diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/SchemaPhysicalTableMatcherSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/SchemaPhysicalTableMatcherSpec.groovy index 1f20f19dd2..0aee3e6c26 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/SchemaPhysicalTableMatcherSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/SchemaPhysicalTableMatcherSpec.groovy @@ -1,4 +1,4 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver @@ -71,14 +71,6 @@ class SchemaPhysicalTableMatcherSpec extends Specification { ['dimA':'druidDimA', 'dimCommon': 'druidDimC', 'dimB': 'dimCommon'] ) - - dimensionDictionary = new DimensionDictionary(dimSet) - schemaPhysicalTableMatcher = new SchemaPhysicalTableMatcher( - request, - query, - DAY.buildZonedTimeGrain(UTC) - ) - request.getGranularity() >> DAY.buildZonedTimeGrain(UTC) query.getInnermostQuery() >> query query.getDimensions() >> (['dimB'] as Set) @@ -86,6 +78,14 @@ class SchemaPhysicalTableMatcherSpec extends Specification { query.getDependentFieldNames() >> ([] as Set) request.getFilterDimensions() >> [] request.getDimensions() >> (dimSet) + request.getFilters() >> Collections.emptyMap() + request.getIntervals() >> [] + request.getLogicalMetrics() >> [] + + dimensionDictionary = new DimensionDictionary(dimSet) + schemaPhysicalTableMatcher = new SchemaPhysicalTableMatcher( + new QueryPlanningConstraint(request, query) + ) } def "schema matcher resolves table containing a logical name for a dimension same as physical name for other dimension"() { diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/VolatileTimeComparatorSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/VolatileTimeComparatorSpec.groovy index 6b5f3ebab4..b8d936f545 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/VolatileTimeComparatorSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/table/resolver/VolatileTimeComparatorSpec.groovy @@ -1,9 +1,10 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.table.resolver import com.yahoo.bard.webservice.data.PartialDataHandler import com.yahoo.bard.webservice.data.QueryBuildingTestingResources +import com.yahoo.bard.webservice.data.metric.TemplateDruidQuery import com.yahoo.bard.webservice.data.time.DefaultTimeGrain import com.yahoo.bard.webservice.druid.model.query.DruidAggregationQuery import com.yahoo.bard.webservice.druid.model.query.Granularity @@ -40,13 +41,14 @@ class VolatileTimeComparatorSpec extends Specification { DefaultTimeGrain.YEAR ) and: "The query of interest" - DruidAggregationQuery query = Stub(DruidAggregationQuery) + TemplateDruidQuery query = Stub(TemplateDruidQuery) + query.getInnermostQuery() >> query + query.getTimeGrain() >> null query.getIntervals() >> (request.getIntervals() as List) and: "The volatile time comparator under test" VolatileTimeComparator comparator = new VolatileTimeComparator( - request, - query, + new QueryPlanningConstraint(request, query), new PartialDataHandler(), resources.volatileIntervalsService ) @@ -93,13 +95,14 @@ class VolatileTimeComparatorSpec extends Specification { requestGranularity ) and: "The query of interest" - DruidAggregationQuery query = Stub(DruidAggregationQuery) + TemplateDruidQuery query = Stub(TemplateDruidQuery) + query.getInnermostQuery() >> query + query.getTimeGrain() >> null query.getIntervals() >> (request.getIntervals() as List) and: "The volatile time comparator under test" VolatileTimeComparator comparator = new VolatileTimeComparator( - request, - query, + new QueryPlanningConstraint(request, query), new PartialDataHandler(), resources.volatileIntervalsService ) diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/handlers/PartialDataRequestHandlerSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/handlers/PartialDataRequestHandlerSpec.groovy index 65f24a8721..4c2538b76f 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/handlers/PartialDataRequestHandlerSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/handlers/PartialDataRequestHandlerSpec.groovy @@ -1,4 +1,4 @@ -// Copyright 2016 Yahoo Inc. +// Copyright 2017 Yahoo Inc. // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.web.handlers @@ -13,6 +13,7 @@ import com.yahoo.bard.webservice.druid.model.query.GroupByQuery import com.yahoo.bard.webservice.table.PhysicalTable import com.yahoo.bard.webservice.table.PhysicalTableDictionary import com.yahoo.bard.webservice.util.SimplifiedIntervalList +import com.yahoo.bard.webservice.util.TableUtils import com.yahoo.bard.webservice.web.DataApiRequest import com.yahoo.bard.webservice.web.responseprocessors.MappingResponseProcessor import com.yahoo.bard.webservice.web.responseprocessors.ResponseContext @@ -47,6 +48,10 @@ class PartialDataRequestHandlerSpec extends Specification { ) def setup() { + apiRequest.getDimensions() >> Collections.emptySet() + apiRequest.getFilterDimensions() >> Collections.emptySet() + groupByQuery.getMetricDimensions() >> Collections.emptySet() + groupByQuery.getDependentFieldNames() >> Collections.emptySet() groupByQuery.getInnermostQuery() >> groupByQuery groupByQuery.getDataSource() >> dataSource physicalTableDictionary.get(_) >> physicalTables[0] @@ -61,7 +66,6 @@ class PartialDataRequestHandlerSpec extends Specification { setup: PARTIAL_DATA.setOn(true) boolean success - ResponseProcessor capture Set intervals = [] as TreeSet apiRequest.intervals >> [] @@ -71,8 +75,7 @@ class PartialDataRequestHandlerSpec extends Specification { then: success 1 * partialDataHandler.findMissingTimeGrainIntervals( - apiRequest, - groupByQuery, + TableUtils.getColumnNames(apiRequest, groupByQuery), physicalTables, new SimplifiedIntervalList(apiRequest.intervals), apiRequest.granularity @@ -104,8 +107,7 @@ class PartialDataRequestHandlerSpec extends Specification { then: success 1 * partialDataHandler.findMissingTimeGrainIntervals( - apiRequest, - groupByQuery, + TableUtils.getColumnNames(apiRequest, groupByQuery), physicalTables, new SimplifiedIntervalList(apiRequest.intervals), apiRequest.granularity @@ -135,7 +137,7 @@ class PartialDataRequestHandlerSpec extends Specification { PARTIAL_DATA.setOn(false) boolean success Interval i = new Interval(0, 1) - SimplifiedIntervalList nonEmptyIntervals = new SimplifiedIntervalList([i]); + SimplifiedIntervalList nonEmptyIntervals = new SimplifiedIntervalList([i]) apiRequest.intervals >> [new Interval(0, 2)] ResponseContext responseContext = new ResponseContext([:]) @@ -150,8 +152,7 @@ class PartialDataRequestHandlerSpec extends Specification { then: success 1 * partialDataHandler.findMissingTimeGrainIntervals( - apiRequest, - groupByQuery, + TableUtils.getColumnNames(apiRequest, groupByQuery), physicalTables, new SimplifiedIntervalList(apiRequest.intervals), apiRequest.granularity