diff --git a/CHANGELOG.md b/CHANGELOG.md index b043c839ba..0b9a9c8a44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,9 @@ Thanks to everyone who contributed to this release! ### Added: +- [Logical Table Availability](https://github.com/yahoo/fili/pull/697) + * Added `logicalTableAvailability` to `TableUtils` which returns the union of availabilities for the logical table. + - [Annotate Functional Interface](https://github.com/yahoo/fili/pull/606) * Add `@FunctionalInterface` annotation to all functional interfaces. diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java b/fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java index fc1dc91f97..477166bf3b 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java @@ -6,6 +6,8 @@ * Feature flags bind an object to a system configuration name. */ public enum BardFeatureFlag implements FeatureFlag { + + CURRENT_MACRO_USES_LATEST("current_macro_uses_latest"), PARTIAL_DATA("partial_data_enabled"), @Deprecated DRUID_CACHE("druid_cache_enabled"), @Deprecated DRUID_CACHE_V2("druid_cache_v2_enabled"), diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java b/fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java index 2cdc57c7bd..6c541e824e 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java @@ -9,6 +9,7 @@ import com.yahoo.bard.webservice.table.resolver.QueryPlanningConstraint; import com.yahoo.bard.webservice.web.DataApiRequest; +import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -95,4 +96,20 @@ public static SimplifiedIntervalList getConstrainedLogicalTableAvailability( .map(PhysicalTable::getAvailableIntervals) .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union); } + + /** + * Returns union of availabilities of the logical table. + * + * @param logicalTable The logical table + * + * @return the union of availabilities of the logical table + */ + public static SimplifiedIntervalList logicalTableAvailability(LogicalTable logicalTable) { + return logicalTable.getTableGroup().getPhysicalTables().stream() + .map(PhysicalTable::getAllAvailableIntervals) + .map(Map::entrySet) + .flatMap(Set::stream) + .map(Map.Entry::getValue) + .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union); + } } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacros.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacro.java similarity index 83% rename from fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacros.java rename to fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacro.java index 00423aa670..553def34ba 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacros.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacro.java @@ -13,11 +13,11 @@ import java.util.stream.Collectors; /** - * TimeMacros. + * TimeMacro. *

* Time macros are used as substitute for the actual date values */ -public enum TimeMacros { +public enum TimeMacro { CURRENT("current", new CurrentMacroCalculation()), NEXT("next", new NextMacroCalculation()); @@ -25,8 +25,8 @@ public enum TimeMacros { private final String macroName; private final MacroCalculationStrategies calculation; - private static final Map NAMES_TO_VALUES = Arrays.stream(TimeMacros.values()) - .collect(Collectors.toMap(TimeMacros::name, Function.identity())); + private static final Map NAMES_TO_VALUES = Arrays.stream(TimeMacro.values()) + .collect(Collectors.toMap(TimeMacro::name, Function.identity())); /** * Constructor. @@ -34,7 +34,7 @@ public enum TimeMacros { * @param macroName Name of the macro * @param calculation Calculation for the macro */ - TimeMacros(String macroName, MacroCalculationStrategies calculation) { + TimeMacro(String macroName, MacroCalculationStrategies calculation) { this.macroName = macroName; this.calculation = calculation; } @@ -55,7 +55,7 @@ public String toString() { * * @return the time macro that matches */ - public static TimeMacros forName(String name) { + public static TimeMacro forName(String name) { return NAMES_TO_VALUES.get(name.toUpperCase(Locale.ENGLISH)); } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java index 397265aa1b..31a7c10616 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java @@ -44,7 +44,7 @@ import com.yahoo.bard.webservice.web.ErrorMessageFormat; import com.yahoo.bard.webservice.web.FilterOperation; import com.yahoo.bard.webservice.web.ResponseFormatType; -import com.yahoo.bard.webservice.web.TimeMacros; +import com.yahoo.bard.webservice.web.TimeMacro; import com.yahoo.bard.webservice.web.filters.ApiFilters; import com.yahoo.bard.webservice.web.util.PaginationLink; import com.yahoo.bard.webservice.web.util.PaginationParameters; @@ -385,6 +385,27 @@ protected static Set generateIntervals( String apiIntervalQuery, Granularity granularity, DateTimeFormatter dateTimeFormatter + ) throws BadApiRequestException { + return generateIntervals(new DateTime(), apiIntervalQuery, granularity, dateTimeFormatter); + } + + + /** + * Extracts the set of intervals from the api request. + * + * @param now The 'now' for which time macros will be relatively calculated + * @param apiIntervalQuery API string containing the intervals in ISO 8601 format, values separated by ','. + * @param granularity The granularity to generate the date based on period or macros. + * @param dateTimeFormatter The formatter to parse date time interval segments + * + * @return Set of jodatime interval objects. + * @throws BadApiRequestException if the requested interval is not found. + */ + protected static Set generateIntervals( + DateTime now, + String apiIntervalQuery, + Granularity granularity, + DateTimeFormatter dateTimeFormatter ) throws BadApiRequestException { try (TimedPhase timer = RequestLog.startTiming("GeneratingIntervals")) { Set generated = new LinkedHashSet<>(); @@ -417,7 +438,6 @@ protected static Set generateIntervals( } Interval interval; - DateTime now = new DateTime(); //If start interval is period, then create new interval with computed end date //possible end interval could be next,current, date if (start.startsWith("P")) { @@ -458,7 +478,6 @@ protected static Set generateIntervals( return generated; } } - /** * Generates filter objects on the based on the filter query in the api request. * @@ -550,7 +569,7 @@ public static DateTime getAsDateTime( DateTimeFormatter timeFormatter ) throws BadApiRequestException { //If granularity is all and dateText is macro, then throw an exception - TimeMacros macro = TimeMacros.forName(dateText); + TimeMacro macro = TimeMacro.forName(dateText); if (macro != null) { if (granularity instanceof AllGranularity) { LOG.debug(INVALID_INTERVAL_GRANULARITY.logFormat(macro, dateText)); diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java index 83561fda00..149a349988 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java @@ -40,6 +40,7 @@ import com.yahoo.bard.webservice.table.LogicalTableDictionary; import com.yahoo.bard.webservice.table.TableIdentifier; import com.yahoo.bard.webservice.util.StreamUtils; +import com.yahoo.bard.webservice.util.TableUtils; import com.yahoo.bard.webservice.web.ApiFilter; import com.yahoo.bard.webservice.web.ApiHaving; import com.yahoo.bard.webservice.web.BadApiRequestException; @@ -294,10 +295,16 @@ public DataApiRequestImpl( LOG.debug(TABLE_UNDEFINED.logFormat(tableName)); throw new BadApiRequestException(TABLE_UNDEFINED.format(tableName)); } - DateTimeFormatter dateTimeFormatter = generateDateTimeFormatter(timeZone); - this.intervals = generateIntervals(intervals, this.granularity, dateTimeFormatter); + if (BardFeatureFlag.CURRENT_MACRO_USES_LATEST.isOn()) { + DateTime firstUnavailableInstant = TableUtils.logicalTableAvailability(getTable()).getLast().getEnd(); + DateTime adjustedNow = firstUnavailableInstant.isBeforeNow() ? firstUnavailableInstant : new DateTime(); + + this.intervals = generateIntervals(adjustedNow, intervals, this.granularity, dateTimeFormatter); + } else { + this.intervals = generateIntervals(intervals, this.granularity, dateTimeFormatter); + } this.filterBuilder = druidFilterBuilder; diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java index d796256d63..5c11d46b4c 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java @@ -3,9 +3,7 @@ package com.yahoo.bard.webservice.web.endpoints; import static com.yahoo.bard.webservice.config.BardFeatureFlag.UPDATED_METADATA_COLLECTION_NAMES; - import static java.util.AbstractMap.SimpleImmutableEntry; - import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; import static javax.ws.rs.core.Response.Status.OK; @@ -18,9 +16,7 @@ import com.yahoo.bard.webservice.logging.blocks.TableRequest; import com.yahoo.bard.webservice.table.LogicalTable; import com.yahoo.bard.webservice.table.LogicalTableDictionary; -import com.yahoo.bard.webservice.table.PhysicalTable; import com.yahoo.bard.webservice.table.resolver.QueryPlanningConstraint; -import com.yahoo.bard.webservice.util.SimplifiedIntervalList; import com.yahoo.bard.webservice.util.TableUtils; import com.yahoo.bard.webservice.web.ErrorMessageFormat; import com.yahoo.bard.webservice.web.RequestMapper; @@ -527,12 +523,7 @@ protected static Map getLogicalTableFullView(LogicalTable logica ); resultRow.put( "availableIntervals", - logicalTable.getTableGroup().getPhysicalTables().stream() - .map(PhysicalTable::getAllAvailableIntervals) - .map(Map::entrySet) - .flatMap(Set::stream) - .map(Map.Entry::getValue) - .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union) + TableUtils.logicalTableAvailability(logicalTable) ); return resultRow; } diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy index 60cf657b36..d2b6e1d013 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy @@ -36,10 +36,11 @@ class FeatureFlagRegistrySpec extends Specification { then: values == ["partial_data_enabled", "druid_cache_enabled", "druid_cache_v2_enabled", "query_split_enabled", - "cache_partial_data", "top_n_enabled", "data_filter_substring_operations_enabled", - "intersection_reporting_enabled", "updated_metadata_collection_names_enabled", - "druid_coordinator_metadata_enabled", "druid_lookup_metadata_enabled", - "druid_dimensions_loader_enabled", "case_sensitive_keys_enabled"] as Set + "cache_partial_data", "top_n_enabled", "current_macro_uses_latest", + "data_filter_substring_operations_enabled", "intersection_reporting_enabled", + "updated_metadata_collection_names_enabled", "druid_coordinator_metadata_enabled", + "druid_lookup_metadata_enabled", "druid_dimensions_loader_enabled", + "case_sensitive_keys_enabled"] as Set } @Unroll diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy index 95264f01d9..3b236d19be 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy @@ -5,6 +5,7 @@ package com.yahoo.bard.webservice.util import com.yahoo.bard.webservice.data.config.names.DataSourceName import com.yahoo.bard.webservice.data.dimension.Dimension import com.yahoo.bard.webservice.druid.model.query.AbstractDruidAggregationQuery +import com.yahoo.bard.webservice.table.Column import com.yahoo.bard.webservice.table.ConfigPhysicalTable import com.yahoo.bard.webservice.table.ConstrainedTable import com.yahoo.bard.webservice.table.LogicalTable @@ -153,4 +154,58 @@ class TableUtilsSpec extends Specification { then: constrainedInterval == SimplifiedIntervalList.simplifyIntervals([constrainedInterval1, constrainedInterval2]) } + + def "logicalTableAvailability returns union of all the intervals for the availability"() { + given: "two intervals [2017, 2018] and [2018, 2019]" + Interval interval1 = new Interval("2017/2018") + Interval interval2 = new Interval("2018/2019") + + AvailabilityTestingUtils.TestAvailability availability1 = new AvailabilityTestingUtils.TestAvailability( + [Mock(DataSourceName)] as Set, + ["availability": [interval1] as Set] + ) + AvailabilityTestingUtils.TestAvailability availability2 = new AvailabilityTestingUtils.TestAvailability( + [Mock(DataSourceName)] as Set, + ["availability": [interval2] as Set] + ) + + PhysicalTableSchema physicalTableSchema = Mock(PhysicalTableSchema) + physicalTableSchema.getPhysicalColumnName(_ as String) >> "" + physicalTableSchema.getColumns() >> Collections.emptySet() + + SimplifiedIntervalList simplifiedIntervalList1 = new SimplifiedIntervalList(Arrays.asList(interval1)) + SimplifiedIntervalList simplifiedIntervalList2 = new SimplifiedIntervalList(Arrays.asList(interval2)) + + Column column1 = Mock(Column) + Column column2 = Mock(Column) + + Map intervalMap1 = new HashMap<>() + intervalMap1.put(column1, simplifiedIntervalList1) + + Map intervalMap2 = new HashMap<>() + intervalMap2.put(column2, simplifiedIntervalList2) + + ConfigPhysicalTable configPhysicalTable1 = Mock(ConfigPhysicalTable) + ConfigPhysicalTable configPhysicalTable2 = Mock(ConfigPhysicalTable) + configPhysicalTable1.getAvailability() >> availability1 + configPhysicalTable1.getAllAvailableIntervals() >> intervalMap1 + configPhysicalTable2.getAllAvailableIntervals() >> intervalMap2 + configPhysicalTable2.getAvailability() >> availability2 + configPhysicalTable1.getSchema() >> physicalTableSchema + configPhysicalTable2.getSchema() >> physicalTableSchema + + TableGroup tableGroup = Mock(TableGroup) + tableGroup.getPhysicalTables() >> ([configPhysicalTable1, configPhysicalTable2] as Set) + + LogicalTable logicalTable = Mock(LogicalTable) + logicalTable.getTableGroup() >> tableGroup + + when: + SimplifiedIntervalList intervals = TableUtils.logicalTableAvailability( + logicalTable + ) + + then: + intervals == SimplifiedIntervalList.simplifyIntervals([interval1, interval2]) + } }