diff --git a/CHANGELOG.md b/CHANGELOG.md
index b043c839ba..0b9a9c8a44 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -145,6 +145,9 @@ Thanks to everyone who contributed to this release!
### Added:
+- [Logical Table Availability](https://github.com/yahoo/fili/pull/697)
+ * Added `logicalTableAvailability` to `TableUtils` which returns the union of availabilities for the logical table.
+
- [Annotate Functional Interface](https://github.com/yahoo/fili/pull/606)
* Add `@FunctionalInterface` annotation to all functional interfaces.
diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java b/fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java
index fc1dc91f97..477166bf3b 100644
--- a/fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java
+++ b/fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java
@@ -6,6 +6,8 @@
* Feature flags bind an object to a system configuration name.
*/
public enum BardFeatureFlag implements FeatureFlag {
+
+ CURRENT_MACRO_USES_LATEST("current_macro_uses_latest"),
PARTIAL_DATA("partial_data_enabled"),
@Deprecated DRUID_CACHE("druid_cache_enabled"),
@Deprecated DRUID_CACHE_V2("druid_cache_v2_enabled"),
diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java b/fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java
index 2cdc57c7bd..6c541e824e 100644
--- a/fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java
+++ b/fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java
@@ -9,6 +9,7 @@
import com.yahoo.bard.webservice.table.resolver.QueryPlanningConstraint;
import com.yahoo.bard.webservice.web.DataApiRequest;
+import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -95,4 +96,20 @@ public static SimplifiedIntervalList getConstrainedLogicalTableAvailability(
.map(PhysicalTable::getAvailableIntervals)
.reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union);
}
+
+ /**
+ * Returns union of availabilities of the logical table.
+ *
+ * @param logicalTable The logical table
+ *
+ * @return the union of availabilities of the logical table
+ */
+ public static SimplifiedIntervalList logicalTableAvailability(LogicalTable logicalTable) {
+ return logicalTable.getTableGroup().getPhysicalTables().stream()
+ .map(PhysicalTable::getAllAvailableIntervals)
+ .map(Map::entrySet)
+ .flatMap(Set::stream)
+ .map(Map.Entry::getValue)
+ .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union);
+ }
}
diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacros.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacro.java
similarity index 83%
rename from fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacros.java
rename to fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacro.java
index 00423aa670..553def34ba 100644
--- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacros.java
+++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacro.java
@@ -13,11 +13,11 @@
import java.util.stream.Collectors;
/**
- * TimeMacros.
+ * TimeMacro.
*
* Time macros are used as substitute for the actual date values
*/
-public enum TimeMacros {
+public enum TimeMacro {
CURRENT("current", new CurrentMacroCalculation()),
NEXT("next", new NextMacroCalculation());
@@ -25,8 +25,8 @@ public enum TimeMacros {
private final String macroName;
private final MacroCalculationStrategies calculation;
- private static final Map NAMES_TO_VALUES = Arrays.stream(TimeMacros.values())
- .collect(Collectors.toMap(TimeMacros::name, Function.identity()));
+ private static final Map NAMES_TO_VALUES = Arrays.stream(TimeMacro.values())
+ .collect(Collectors.toMap(TimeMacro::name, Function.identity()));
/**
* Constructor.
@@ -34,7 +34,7 @@ public enum TimeMacros {
* @param macroName Name of the macro
* @param calculation Calculation for the macro
*/
- TimeMacros(String macroName, MacroCalculationStrategies calculation) {
+ TimeMacro(String macroName, MacroCalculationStrategies calculation) {
this.macroName = macroName;
this.calculation = calculation;
}
@@ -55,7 +55,7 @@ public String toString() {
*
* @return the time macro that matches
*/
- public static TimeMacros forName(String name) {
+ public static TimeMacro forName(String name) {
return NAMES_TO_VALUES.get(name.toUpperCase(Locale.ENGLISH));
}
diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java
index 397265aa1b..31a7c10616 100644
--- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java
+++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java
@@ -44,7 +44,7 @@
import com.yahoo.bard.webservice.web.ErrorMessageFormat;
import com.yahoo.bard.webservice.web.FilterOperation;
import com.yahoo.bard.webservice.web.ResponseFormatType;
-import com.yahoo.bard.webservice.web.TimeMacros;
+import com.yahoo.bard.webservice.web.TimeMacro;
import com.yahoo.bard.webservice.web.filters.ApiFilters;
import com.yahoo.bard.webservice.web.util.PaginationLink;
import com.yahoo.bard.webservice.web.util.PaginationParameters;
@@ -385,6 +385,27 @@ protected static Set generateIntervals(
String apiIntervalQuery,
Granularity granularity,
DateTimeFormatter dateTimeFormatter
+ ) throws BadApiRequestException {
+ return generateIntervals(new DateTime(), apiIntervalQuery, granularity, dateTimeFormatter);
+ }
+
+
+ /**
+ * Extracts the set of intervals from the api request.
+ *
+ * @param now The 'now' for which time macros will be relatively calculated
+ * @param apiIntervalQuery API string containing the intervals in ISO 8601 format, values separated by ','.
+ * @param granularity The granularity to generate the date based on period or macros.
+ * @param dateTimeFormatter The formatter to parse date time interval segments
+ *
+ * @return Set of jodatime interval objects.
+ * @throws BadApiRequestException if the requested interval is not found.
+ */
+ protected static Set generateIntervals(
+ DateTime now,
+ String apiIntervalQuery,
+ Granularity granularity,
+ DateTimeFormatter dateTimeFormatter
) throws BadApiRequestException {
try (TimedPhase timer = RequestLog.startTiming("GeneratingIntervals")) {
Set generated = new LinkedHashSet<>();
@@ -417,7 +438,6 @@ protected static Set generateIntervals(
}
Interval interval;
- DateTime now = new DateTime();
//If start interval is period, then create new interval with computed end date
//possible end interval could be next,current, date
if (start.startsWith("P")) {
@@ -458,7 +478,6 @@ protected static Set generateIntervals(
return generated;
}
}
-
/**
* Generates filter objects on the based on the filter query in the api request.
*
@@ -550,7 +569,7 @@ public static DateTime getAsDateTime(
DateTimeFormatter timeFormatter
) throws BadApiRequestException {
//If granularity is all and dateText is macro, then throw an exception
- TimeMacros macro = TimeMacros.forName(dateText);
+ TimeMacro macro = TimeMacro.forName(dateText);
if (macro != null) {
if (granularity instanceof AllGranularity) {
LOG.debug(INVALID_INTERVAL_GRANULARITY.logFormat(macro, dateText));
diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java
index 83561fda00..149a349988 100644
--- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java
+++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java
@@ -40,6 +40,7 @@
import com.yahoo.bard.webservice.table.LogicalTableDictionary;
import com.yahoo.bard.webservice.table.TableIdentifier;
import com.yahoo.bard.webservice.util.StreamUtils;
+import com.yahoo.bard.webservice.util.TableUtils;
import com.yahoo.bard.webservice.web.ApiFilter;
import com.yahoo.bard.webservice.web.ApiHaving;
import com.yahoo.bard.webservice.web.BadApiRequestException;
@@ -294,10 +295,16 @@ public DataApiRequestImpl(
LOG.debug(TABLE_UNDEFINED.logFormat(tableName));
throw new BadApiRequestException(TABLE_UNDEFINED.format(tableName));
}
-
DateTimeFormatter dateTimeFormatter = generateDateTimeFormatter(timeZone);
- this.intervals = generateIntervals(intervals, this.granularity, dateTimeFormatter);
+ if (BardFeatureFlag.CURRENT_MACRO_USES_LATEST.isOn()) {
+ DateTime firstUnavailableInstant = TableUtils.logicalTableAvailability(getTable()).getLast().getEnd();
+ DateTime adjustedNow = firstUnavailableInstant.isBeforeNow() ? firstUnavailableInstant : new DateTime();
+
+ this.intervals = generateIntervals(adjustedNow, intervals, this.granularity, dateTimeFormatter);
+ } else {
+ this.intervals = generateIntervals(intervals, this.granularity, dateTimeFormatter);
+ }
this.filterBuilder = druidFilterBuilder;
diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java
index d796256d63..5c11d46b4c 100644
--- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java
+++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java
@@ -3,9 +3,7 @@
package com.yahoo.bard.webservice.web.endpoints;
import static com.yahoo.bard.webservice.config.BardFeatureFlag.UPDATED_METADATA_COLLECTION_NAMES;
-
import static java.util.AbstractMap.SimpleImmutableEntry;
-
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
import static javax.ws.rs.core.Response.Status.OK;
@@ -18,9 +16,7 @@
import com.yahoo.bard.webservice.logging.blocks.TableRequest;
import com.yahoo.bard.webservice.table.LogicalTable;
import com.yahoo.bard.webservice.table.LogicalTableDictionary;
-import com.yahoo.bard.webservice.table.PhysicalTable;
import com.yahoo.bard.webservice.table.resolver.QueryPlanningConstraint;
-import com.yahoo.bard.webservice.util.SimplifiedIntervalList;
import com.yahoo.bard.webservice.util.TableUtils;
import com.yahoo.bard.webservice.web.ErrorMessageFormat;
import com.yahoo.bard.webservice.web.RequestMapper;
@@ -527,12 +523,7 @@ protected static Map getLogicalTableFullView(LogicalTable logica
);
resultRow.put(
"availableIntervals",
- logicalTable.getTableGroup().getPhysicalTables().stream()
- .map(PhysicalTable::getAllAvailableIntervals)
- .map(Map::entrySet)
- .flatMap(Set::stream)
- .map(Map.Entry::getValue)
- .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union)
+ TableUtils.logicalTableAvailability(logicalTable)
);
return resultRow;
}
diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy
index 60cf657b36..d2b6e1d013 100644
--- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy
+++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy
@@ -36,10 +36,11 @@ class FeatureFlagRegistrySpec extends Specification {
then:
values == ["partial_data_enabled", "druid_cache_enabled", "druid_cache_v2_enabled", "query_split_enabled",
- "cache_partial_data", "top_n_enabled", "data_filter_substring_operations_enabled",
- "intersection_reporting_enabled", "updated_metadata_collection_names_enabled",
- "druid_coordinator_metadata_enabled", "druid_lookup_metadata_enabled",
- "druid_dimensions_loader_enabled", "case_sensitive_keys_enabled"] as Set
+ "cache_partial_data", "top_n_enabled", "current_macro_uses_latest",
+ "data_filter_substring_operations_enabled", "intersection_reporting_enabled",
+ "updated_metadata_collection_names_enabled", "druid_coordinator_metadata_enabled",
+ "druid_lookup_metadata_enabled", "druid_dimensions_loader_enabled",
+ "case_sensitive_keys_enabled"] as Set
}
@Unroll
diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy
index 95264f01d9..3b236d19be 100644
--- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy
+++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy
@@ -5,6 +5,7 @@ package com.yahoo.bard.webservice.util
import com.yahoo.bard.webservice.data.config.names.DataSourceName
import com.yahoo.bard.webservice.data.dimension.Dimension
import com.yahoo.bard.webservice.druid.model.query.AbstractDruidAggregationQuery
+import com.yahoo.bard.webservice.table.Column
import com.yahoo.bard.webservice.table.ConfigPhysicalTable
import com.yahoo.bard.webservice.table.ConstrainedTable
import com.yahoo.bard.webservice.table.LogicalTable
@@ -153,4 +154,58 @@ class TableUtilsSpec extends Specification {
then:
constrainedInterval == SimplifiedIntervalList.simplifyIntervals([constrainedInterval1, constrainedInterval2])
}
+
+ def "logicalTableAvailability returns union of all the intervals for the availability"() {
+ given: "two intervals [2017, 2018] and [2018, 2019]"
+ Interval interval1 = new Interval("2017/2018")
+ Interval interval2 = new Interval("2018/2019")
+
+ AvailabilityTestingUtils.TestAvailability availability1 = new AvailabilityTestingUtils.TestAvailability(
+ [Mock(DataSourceName)] as Set,
+ ["availability": [interval1] as Set]
+ )
+ AvailabilityTestingUtils.TestAvailability availability2 = new AvailabilityTestingUtils.TestAvailability(
+ [Mock(DataSourceName)] as Set,
+ ["availability": [interval2] as Set]
+ )
+
+ PhysicalTableSchema physicalTableSchema = Mock(PhysicalTableSchema)
+ physicalTableSchema.getPhysicalColumnName(_ as String) >> ""
+ physicalTableSchema.getColumns() >> Collections.emptySet()
+
+ SimplifiedIntervalList simplifiedIntervalList1 = new SimplifiedIntervalList(Arrays.asList(interval1))
+ SimplifiedIntervalList simplifiedIntervalList2 = new SimplifiedIntervalList(Arrays.asList(interval2))
+
+ Column column1 = Mock(Column)
+ Column column2 = Mock(Column)
+
+ Map intervalMap1 = new HashMap<>()
+ intervalMap1.put(column1, simplifiedIntervalList1)
+
+ Map intervalMap2 = new HashMap<>()
+ intervalMap2.put(column2, simplifiedIntervalList2)
+
+ ConfigPhysicalTable configPhysicalTable1 = Mock(ConfigPhysicalTable)
+ ConfigPhysicalTable configPhysicalTable2 = Mock(ConfigPhysicalTable)
+ configPhysicalTable1.getAvailability() >> availability1
+ configPhysicalTable1.getAllAvailableIntervals() >> intervalMap1
+ configPhysicalTable2.getAllAvailableIntervals() >> intervalMap2
+ configPhysicalTable2.getAvailability() >> availability2
+ configPhysicalTable1.getSchema() >> physicalTableSchema
+ configPhysicalTable2.getSchema() >> physicalTableSchema
+
+ TableGroup tableGroup = Mock(TableGroup)
+ tableGroup.getPhysicalTables() >> ([configPhysicalTable1, configPhysicalTable2] as Set)
+
+ LogicalTable logicalTable = Mock(LogicalTable)
+ logicalTable.getTableGroup() >> tableGroup
+
+ when:
+ SimplifiedIntervalList intervals = TableUtils.logicalTableAvailability(
+ logicalTable
+ )
+
+ then:
+ intervals == SimplifiedIntervalList.simplifyIntervals([interval1, interval2])
+ }
}