Latest time macro

CHANGELOG.md

@@ -145,6 +145,9 @@ Thanks to everyone who contributed to this release!
 
 ### Added:
 
+- [Logical Table Availability](https://github.com/yahoo/fili/pull/697)
+    * Added `logicalTableAvailability` to `TableUtils` which returns the union of availabilities for the logical table.
+
 - [Annotate Functional Interface](https://github.com/yahoo/fili/pull/606)
     * Add `@FunctionalInterface` annotation to all functional interfaces.
 

fili-core/src/main/java/com/yahoo/bard/webservice/config/BardFeatureFlag.java

@@ -6,6 +6,8 @@
  * Feature flags bind an object to a system configuration name.
  */
 public enum BardFeatureFlag implements FeatureFlag {
+
+    CURRENT_MACRO_USES_LATEST("current_macro_uses_latest"),
     PARTIAL_DATA("partial_data_enabled"),
     @Deprecated DRUID_CACHE("druid_cache_enabled"),
     @Deprecated DRUID_CACHE_V2("druid_cache_v2_enabled"),

fili-core/src/main/java/com/yahoo/bard/webservice/util/TableUtils.java

@@ -9,6 +9,7 @@
 import com.yahoo.bard.webservice.table.resolver.QueryPlanningConstraint;
 import com.yahoo.bard.webservice.web.DataApiRequest;
 
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -95,4 +96,20 @@ public static SimplifiedIntervalList getConstrainedLogicalTableAvailability(
                 .map(PhysicalTable::getAvailableIntervals)
                 .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union);
     }
+
+    /**
+     * Returns union of availabilities of the logical table.
+     *
+     * @param logicalTable  The logical table
+     *
+     * @return the union of availabilities of the logical table
+     */
+    public static SimplifiedIntervalList logicalTableAvailability(LogicalTable logicalTable) {
+        return logicalTable.getTableGroup().getPhysicalTables().stream()
+                .map(PhysicalTable::getAllAvailableIntervals)
+                .map(Map::entrySet)
+                .flatMap(Set::stream)
+                .map(Map.Entry::getValue)
+                .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union);
+    }
 }

fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacros.java → fili-core/src/main/java/com/yahoo/bard/webservice/web/TimeMacro.java

@@ -13,28 +13,28 @@
 import java.util.stream.Collectors;
 
 /**
- * TimeMacros.
+ * TimeMacro.
  * <p>
  * Time macros are used as substitute for the actual date values
  */
-public enum TimeMacros {
+public enum TimeMacro {
 
     CURRENT("current", new CurrentMacroCalculation()),
     NEXT("next", new NextMacroCalculation());
 
     private final String macroName;
     private final MacroCalculationStrategies calculation;
 
-    private static final Map<String, TimeMacros> NAMES_TO_VALUES = Arrays.stream(TimeMacros.values())
-            .collect(Collectors.toMap(TimeMacros::name, Function.identity()));
+    private static final Map<String, TimeMacro> NAMES_TO_VALUES = Arrays.stream(TimeMacro.values())
+            .collect(Collectors.toMap(TimeMacro::name, Function.identity()));
 
     /**
      * Constructor.
      *
      * @param macroName  Name of the macro
      * @param calculation  Calculation for the macro
      */
-    TimeMacros(String macroName, MacroCalculationStrategies calculation) {
+    TimeMacro(String macroName, MacroCalculationStrategies calculation) {
         this.macroName = macroName;
         this.calculation = calculation;
     }
@@ -55,7 +55,7 @@ public String toString() {
      *
      * @return the time macro that matches
      */
-    public static TimeMacros forName(String name) {
+    public static TimeMacro forName(String name) {
         return NAMES_TO_VALUES.get(name.toUpperCase(Locale.ENGLISH));
     }
 

fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/ApiRequestImpl.java

@@ -44,7 +44,7 @@
 import com.yahoo.bard.webservice.web.ErrorMessageFormat;
 import com.yahoo.bard.webservice.web.FilterOperation;
 import com.yahoo.bard.webservice.web.ResponseFormatType;
-import com.yahoo.bard.webservice.web.TimeMacros;
+import com.yahoo.bard.webservice.web.TimeMacro;
 import com.yahoo.bard.webservice.web.filters.ApiFilters;
 import com.yahoo.bard.webservice.web.util.PaginationLink;
 import com.yahoo.bard.webservice.web.util.PaginationParameters;
@@ -385,6 +385,27 @@ protected static Set<Interval> generateIntervals(
             String apiIntervalQuery,
             Granularity granularity,
             DateTimeFormatter dateTimeFormatter
+    ) throws BadApiRequestException {
+        return generateIntervals(new DateTime(), apiIntervalQuery, granularity, dateTimeFormatter);
+    }
+
+
+    /**
+     * Extracts the set of intervals from the api request.
+     *
+     * @param now The 'now' for which time macros will be relatively calculated
+     * @param apiIntervalQuery  API string containing the intervals in ISO 8601 format, values separated by ','.
+     * @param granularity  The granularity to generate the date based on period or macros.
+     * @param dateTimeFormatter  The formatter to parse date time interval segments
+     *
+     * @return Set of jodatime interval objects.
+     * @throws BadApiRequestException if the requested interval is not found.
+     */
+    protected static Set<Interval> generateIntervals(
+            DateTime now,
+            String apiIntervalQuery,
+            Granularity granularity,
+            DateTimeFormatter dateTimeFormatter
     ) throws BadApiRequestException {
         try (TimedPhase timer = RequestLog.startTiming("GeneratingIntervals")) {
             Set<Interval> generated = new LinkedHashSet<>();
@@ -417,7 +438,6 @@ protected static Set<Interval> generateIntervals(
                     }
 
                     Interval interval;
-                    DateTime now = new DateTime();
                     //If start interval is period, then create new interval with computed end date
                     //possible end interval could be next,current, date
                     if (start.startsWith("P")) {
@@ -458,7 +478,6 @@ protected static Set<Interval> generateIntervals(
             return generated;
         }
     }
-
     /**
      * Generates filter objects on the based on the filter query in the api request.
      *
@@ -550,7 +569,7 @@ public static DateTime getAsDateTime(
             DateTimeFormatter timeFormatter
     ) throws BadApiRequestException {
         //If granularity is all and dateText is macro, then throw an exception
-        TimeMacros macro = TimeMacros.forName(dateText);
+        TimeMacro macro = TimeMacro.forName(dateText);
         if (macro != null) {
             if (granularity instanceof AllGranularity) {
                 LOG.debug(INVALID_INTERVAL_GRANULARITY.logFormat(macro, dateText));

fili-core/src/main/java/com/yahoo/bard/webservice/web/apirequest/DataApiRequestImpl.java

@@ -40,6 +40,7 @@
 import com.yahoo.bard.webservice.table.LogicalTableDictionary;
 import com.yahoo.bard.webservice.table.TableIdentifier;
 import com.yahoo.bard.webservice.util.StreamUtils;
+import com.yahoo.bard.webservice.util.TableUtils;
 import com.yahoo.bard.webservice.web.ApiFilter;
 import com.yahoo.bard.webservice.web.ApiHaving;
 import com.yahoo.bard.webservice.web.BadApiRequestException;
@@ -294,10 +295,16 @@ public DataApiRequestImpl(
             LOG.debug(TABLE_UNDEFINED.logFormat(tableName));
             throw new BadApiRequestException(TABLE_UNDEFINED.format(tableName));
         }
-
         DateTimeFormatter dateTimeFormatter = generateDateTimeFormatter(timeZone);
 
-        this.intervals = generateIntervals(intervals, this.granularity, dateTimeFormatter);
+        if (BardFeatureFlag.CURRENT_MACRO_USES_LATEST.isOn()) {
+            DateTime firstUnavailableInstant = TableUtils.logicalTableAvailability(getTable()).getLast().getEnd();
+            DateTime adjustedNow =  firstUnavailableInstant.isBeforeNow() ? firstUnavailableInstant : new DateTime();
+
+            this.intervals = generateIntervals(adjustedNow, intervals, this.granularity, dateTimeFormatter);
+        } else {
+            this.intervals = generateIntervals(intervals, this.granularity, dateTimeFormatter);
+        }
 
         this.filterBuilder = druidFilterBuilder;
 

fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/TablesServlet.java

@@ -3,9 +3,7 @@
 package com.yahoo.bard.webservice.web.endpoints;
 
 import static com.yahoo.bard.webservice.config.BardFeatureFlag.UPDATED_METADATA_COLLECTION_NAMES;
-
 import static java.util.AbstractMap.SimpleImmutableEntry;
-
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
 import static javax.ws.rs.core.Response.Status.OK;
@@ -18,9 +16,7 @@
 import com.yahoo.bard.webservice.logging.blocks.TableRequest;
 import com.yahoo.bard.webservice.table.LogicalTable;
 import com.yahoo.bard.webservice.table.LogicalTableDictionary;
-import com.yahoo.bard.webservice.table.PhysicalTable;
 import com.yahoo.bard.webservice.table.resolver.QueryPlanningConstraint;
-import com.yahoo.bard.webservice.util.SimplifiedIntervalList;
 import com.yahoo.bard.webservice.util.TableUtils;
 import com.yahoo.bard.webservice.web.ErrorMessageFormat;
 import com.yahoo.bard.webservice.web.RequestMapper;
@@ -527,12 +523,7 @@ protected static Map<String, Object> getLogicalTableFullView(LogicalTable logica
         );
         resultRow.put(
                 "availableIntervals",
-                logicalTable.getTableGroup().getPhysicalTables().stream()
-                        .map(PhysicalTable::getAllAvailableIntervals)
-                        .map(Map::entrySet)
-                        .flatMap(Set::stream)
-                        .map(Map.Entry::getValue)
-                        .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union)
+                TableUtils.logicalTableAvailability(logicalTable)
         );
         return resultRow;
     }

fili-core/src/test/groovy/com/yahoo/bard/webservice/config/FeatureFlagRegistrySpec.groovy

@@ -36,10 +36,11 @@ class FeatureFlagRegistrySpec extends Specification {
 
         then:
         values == ["partial_data_enabled", "druid_cache_enabled", "druid_cache_v2_enabled", "query_split_enabled",
-                   "cache_partial_data", "top_n_enabled", "data_filter_substring_operations_enabled",
-                   "intersection_reporting_enabled", "updated_metadata_collection_names_enabled",
-                   "druid_coordinator_metadata_enabled", "druid_lookup_metadata_enabled",
-                   "druid_dimensions_loader_enabled", "case_sensitive_keys_enabled"] as Set
+                   "cache_partial_data", "top_n_enabled", "current_macro_uses_latest",
+                   "data_filter_substring_operations_enabled", "intersection_reporting_enabled",
+                   "updated_metadata_collection_names_enabled", "druid_coordinator_metadata_enabled",
+                   "druid_lookup_metadata_enabled", "druid_dimensions_loader_enabled",
+                   "case_sensitive_keys_enabled"] as Set
     }
 
     @Unroll

fili-core/src/test/groovy/com/yahoo/bard/webservice/util/TableUtilsSpec.groovy

@@ -5,6 +5,7 @@ package com.yahoo.bard.webservice.util
 import com.yahoo.bard.webservice.data.config.names.DataSourceName
 import com.yahoo.bard.webservice.data.dimension.Dimension
 import com.yahoo.bard.webservice.druid.model.query.AbstractDruidAggregationQuery
+import com.yahoo.bard.webservice.table.Column
 import com.yahoo.bard.webservice.table.ConfigPhysicalTable
 import com.yahoo.bard.webservice.table.ConstrainedTable
 import com.yahoo.bard.webservice.table.LogicalTable
@@ -153,4 +154,58 @@ class TableUtilsSpec extends  Specification {
         then:
         constrainedInterval == SimplifiedIntervalList.simplifyIntervals([constrainedInterval1, constrainedInterval2])
     }
+
+    def "logicalTableAvailability returns union of all the intervals for the availability"() {
+        given: "two intervals [2017, 2018] and [2018, 2019]"
+        Interval interval1 = new Interval("2017/2018")
+        Interval interval2 = new Interval("2018/2019")
+
+        AvailabilityTestingUtils.TestAvailability availability1 = new AvailabilityTestingUtils.TestAvailability(
+                [Mock(DataSourceName)] as Set,
+                ["availability": [interval1] as Set]
+        )
+        AvailabilityTestingUtils.TestAvailability availability2 = new AvailabilityTestingUtils.TestAvailability(
+                [Mock(DataSourceName)] as Set,
+                ["availability": [interval2] as Set]
+        )
+
+        PhysicalTableSchema physicalTableSchema = Mock(PhysicalTableSchema)
+        physicalTableSchema.getPhysicalColumnName(_ as String) >> ""
+        physicalTableSchema.getColumns() >> Collections.emptySet()
+
+        SimplifiedIntervalList simplifiedIntervalList1 = new SimplifiedIntervalList(Arrays.asList(interval1))
+        SimplifiedIntervalList simplifiedIntervalList2 = new SimplifiedIntervalList(Arrays.asList(interval2))
+
+        Column column1 = Mock(Column)
+        Column column2 = Mock(Column)
+
+        Map<Column, SimplifiedIntervalList> intervalMap1 = new HashMap<>()
+        intervalMap1.put(column1, simplifiedIntervalList1)
+
+        Map<Column, SimplifiedIntervalList> intervalMap2 = new HashMap<>()
+        intervalMap2.put(column2, simplifiedIntervalList2)
+
+        ConfigPhysicalTable configPhysicalTable1 = Mock(ConfigPhysicalTable)
+        ConfigPhysicalTable configPhysicalTable2 = Mock(ConfigPhysicalTable)
+        configPhysicalTable1.getAvailability() >> availability1
+        configPhysicalTable1.getAllAvailableIntervals() >> intervalMap1
+        configPhysicalTable2.getAllAvailableIntervals() >> intervalMap2
+        configPhysicalTable2.getAvailability() >> availability2
+        configPhysicalTable1.getSchema() >> physicalTableSchema
+        configPhysicalTable2.getSchema() >> physicalTableSchema
+
+        TableGroup tableGroup = Mock(TableGroup)
+        tableGroup.getPhysicalTables() >> ([configPhysicalTable1, configPhysicalTable2] as Set)
+
+        LogicalTable logicalTable = Mock(LogicalTable)
+        logicalTable.getTableGroup() >> tableGroup
+
+        when:
+        SimplifiedIntervalList intervals = TableUtils.logicalTableAvailability(
+                logicalTable
+        )
+
+        then:
+        intervals == SimplifiedIntervalList.simplifyIntervals([interval1, interval2])
+    }
 }