From 781c4f31bed89f5234d6cf9e0f7704324c58f968 Mon Sep 17 00:00:00 2001 From: Carlos Devoto Date: Wed, 9 Dec 2015 18:40:32 -0500 Subject: [PATCH] Align downsampling intervals to the Gregorian calendar. This feature supports the alignment of downsampling intervals to the Gregorian calendar based on four different time categories: - DAILY: The start time of each interval is computed as the start of the day in which the first data point occurs, based on a specified time zone (or the default JVM time zone, if no time zone has been specified). The end time of each interval is computed as the end of the day in which the first data point occurs. For instance, if the specified time zone is UTC, and the timestamp of the first data point is 2016-01-05T05:32:00Z, then start of the interval will be computed as 2016-01-05T00:00:00.000Z, while the end of the interval will be computed as 2016-01-05T23:59:59.999Z. - WEEKLY: The start time of each interval is computed as the start of the week in which the first data point occurs, based on a specified time zone (or the default JVM time zone, if no time zone has been specified). The end time of each interval is computed as the end of the week in which the first data point occurs. Weeks are considered to begin on Sundays (in the future, it might be a good idea to allow for variations based on a configuration setting). For instance, if the specified time zone is UTC, and the timestamp of the first data point is 2016-01-05T05:32:00Z, then start of the interval will be computed as 2016-01-03T00:00:00.000Z, while the end of the interval will be computed as 2016-01-09T23:59:59.999Z. - MONTHLY: The start time of each interval is computed as the start of the month in which the first data point occurs, based on a specified time zone (or the default JVM time zone, if no time zone has been specified). The end time of each interval is computed as the end of the month in which the first data point occurs. For instance, if the specified time zone is UTC, and the timestamp of the first data point is 2016-01-05T05:32:00Z, then start of the interval will be computed as 2016-01-01T00:00:00.000Z, while the end of the interval will be computed as 2016-01-31T23:59:59.999Z. - YEARLY: The start time of each interval is computed as the start of the year in which the first data point occurs, based on a specified time zone (or the default JVM time zone, if no time zone has been specified). The end time of each interval is computed as the end of the year in which the first data point occurs. For instance, if the specified time zone is UTC, and the timestamp of the first data point is 2016-01-05T05:32:00Z, then start of the interval will be computed as 2016-01-01T00:00:00.000Z, while the end of the interval will be computed as 2016-12-31T23:59:59.999Z. This feature also allows for the alignment of intervals that are multiples of one year, one month, one week, or one day. In cases where a given interval is a multiple of more than one time category, the larger time category will be used. For instance, an interval of 24 months will be interpreted as a interval of two years, and will be aligned to the calendar accordingly. As such, if the specified time zone is UTC, and the timestamp of the first data point is 2016-03-05T05:32:00Z, then start of the interval will be computed as 2016-01-01T00:00:00.000Z, while the end of the interval will be computed as 2017-12-31T23:59:59.999Z. This is in keeping with the principle of least astonishment. To specify the time zone for a given HTTP query, include a query string parameter named "tz" with a value equal to a JVM time zone id (e.g. "UTC"). If a time zone is not included in the query string, the default JVM time zone will be used. To specify that a given HTTP query should use the calendar alignment feature for downsampling, include a query string parameter named "use_calendar" with a value of "true". You can stipulate that all HTTP queries should use the calendar alignment feature by including a "tsd.query.downsample.use_calendar" configuration setting within the opentsdb.conf file and by setting its value to "true" (the default value is "false"). This config file setting can be overridden on a per-query basis by including the "use_calendar" parameter in the query string as specified above. --- src/core/AggregationIterator.java | 51 +++- src/core/Downsampler.java | 166 +++++++++- src/core/FillingDownsampler.java | 27 +- src/core/Query.java | 20 +- src/core/Span.java | 25 +- src/core/SpanGroup.java | 67 ++++- src/core/TSQuery.java | 13 + src/core/TsdbQuery.java | 64 +++- src/tsd/QueryRpc.java | 17 ++ src/utils/Config.java | 11 + src/utils/DateTime.java | 171 +++++++++++ test/core/TestDownsampler.java | 483 +++++++++++++++++++++++++++++- test/core/TestSpan.java | 5 +- test/tsd/TestQueryRpc.java | 56 +++- test/utils/TestDateTime.java | 46 +++ 15 files changed, 1175 insertions(+), 47 deletions(-) diff --git a/src/core/AggregationIterator.java b/src/core/AggregationIterator.java index 28d77b9939..fb2a97db29 100644 --- a/src/core/AggregationIterator.java +++ b/src/core/AggregationIterator.java @@ -15,13 +15,16 @@ import java.util.Arrays; import java.util.List; import java.util.NoSuchElementException; - -import com.google.common.annotations.VisibleForTesting; +import java.util.TimeZone; import net.opentsdb.core.Aggregators.Interpolation; +import net.opentsdb.utils.DateTime; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + /** * Iterator that aggregates multiple spans or time series data and does linear * interpolation (lerp) for missing data points. @@ -225,6 +228,41 @@ public static AggregationIterator create(final List spans, return create(spans, start_time, end_time, aggregator, method, downsampler, sample_interval_ms, rate, rate_options, null); } + + /** + * Creates a new iterator for a {@link SpanGroup}. + * @param spans Spans in a group. + * @param start_time Any data point strictly before this timestamp will be + * ignored. + * @param end_time Any data point strictly after this timestamp will be + * ignored. + * @param aggregator The aggregation function to use. + * @param method Interpolation method to use when aggregating time series + * @param downsampler Aggregation function to use to group data points + * within an interval. + * @param sample_interval_ms Number of milliseconds wanted between each data + * point. + * @param rate If {@code true}, the rate of the series will be used instead + * of the actual values. + * @param rate_options Specifies the optional additional rate calculation + * options. + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @return An {@link AggregationIterator} object. + */ + public static AggregationIterator create(final List spans, + final long start_time, + final long end_time, + final Aggregator aggregator, + final Interpolation method, + final Aggregator downsampler, + final long sample_interval_ms, + final boolean rate, + final RateOptions rate_options, + final FillPolicy fill_policy) { + return create(spans, start_time, end_time, aggregator, method, downsampler, + sample_interval_ms, rate, rate_options, fill_policy, null, false); + } /** * Creates a new iterator for a {@link SpanGroup}. @@ -245,6 +283,8 @@ public static AggregationIterator create(final List spans, * options. * @param fill_policy Policy specifying whether to interpolate or to fill * missing intervals with special values. + * @param timezone The timezone to use for aligning intervals based on the calendar. + * @param use_calendar A flag denoting whether or not to align intervals based on the calendar. * @return An {@link AggregationIterator} object. * @since 2.2 */ @@ -257,16 +297,19 @@ public static AggregationIterator create(final List spans, final long sample_interval_ms, final boolean rate, final RateOptions rate_options, - final FillPolicy fill_policy) { + final FillPolicy fill_policy, + final String timezone, + final boolean use_calendar) { final int size = spans.size(); final SeekableView[] iterators = new SeekableView[size]; + TimeZone tz = DateTime.timezones.get(timezone); for (int i = 0; i < size; i++) { SeekableView it; if (downsampler == null) { it = spans.get(i).spanIterator(); } else { it = spans.get(i).downsampler(start_time, end_time, sample_interval_ms, - downsampler, fill_policy); + downsampler, fill_policy, tz, use_calendar); } if (rate) { it = new RateSpan(it, rate_options); diff --git a/src/core/Downsampler.java b/src/core/Downsampler.java index d4d56ff51d..eb0da5f2fe 100644 --- a/src/core/Downsampler.java +++ b/src/core/Downsampler.java @@ -12,13 +12,28 @@ // see . package net.opentsdb.core; +import static net.opentsdb.utils.DateTime.toEndOfDay; +import static net.opentsdb.utils.DateTime.toEndOfMonth; +import static net.opentsdb.utils.DateTime.toEndOfWeek; +import static net.opentsdb.utils.DateTime.toEndOfYear; +import static net.opentsdb.utils.DateTime.toStartOfDay; +import static net.opentsdb.utils.DateTime.toStartOfMonth; +import static net.opentsdb.utils.DateTime.toStartOfWeek; +import static net.opentsdb.utils.DateTime.toStartOfYear; + import java.util.NoSuchElementException; +import java.util.TimeZone; /** * Iterator that downsamples data points using an {@link Aggregator}. */ public class Downsampler implements SeekableView, DataPoint { + static final long ONE_WEEK_INTERVAL = 604800000L; + static final long ONE_MONTH_INTERVAL = 2592000000L; + static final long ONE_YEAR_INTERVAL = 31536000000L; + static final long ONE_DAY_INTERVAL = 86400000L; + /** Function to use for downsampling. */ protected final Aggregator downsampler; /** Iterator to iterate the values of the current interval. */ @@ -29,7 +44,7 @@ public class Downsampler implements SeekableView, DataPoint { protected double value; /** - * Ctor. + * Ctor (for backward compatibility). * @param source The iterator to access the underlying data. * @param interval_ms The interval in milli seconds wanted between each data * point. @@ -38,10 +53,26 @@ public class Downsampler implements SeekableView, DataPoint { Downsampler(final SeekableView source, final long interval_ms, final Aggregator downsampler) { - this.values_in_interval = new ValuesInInterval(source, interval_ms); - this.downsampler = downsampler; + this(source, interval_ms, downsampler, null, false); } + /** + * Ctor. + * @param source The iterator to access the underlying data. + * @param interval_ms The interval in milli seconds wanted between each data + * point. + * @param downsampler The downsampling function to use. + * @param timezone The timezone to use for aligning intervals based on the calendar. + * @param use_calendar A flag denoting whether or not to align intervals based on the calendar. + */ + Downsampler(final SeekableView source, + final long interval_ms, + final Aggregator downsampler, + final TimeZone timezone, + final boolean use_calendar) { + this.values_in_interval = new ValuesInInterval(source, interval_ms, timezone, use_calendar); + this.downsampler = downsampler; + } // ------------------ // // Iterator interface // // ------------------ // @@ -133,6 +164,10 @@ protected static class ValuesInInterval implements Aggregator.Doubles { private boolean has_next_value_from_source = false; /** The last data point extracted from the source. */ private DataPoint next_dp = null; + /** The timezone to use for aligning intervals based on the calendar */ + private final TimeZone timezone; + /** A flag denoting whether or not to align intervals based on the calendar */ + private final boolean use_calendar; /** True if it is initialized for iterating intervals. */ private boolean initialized = false; @@ -142,10 +177,13 @@ protected static class ValuesInInterval implements Aggregator.Doubles { * @param source The iterator to access the underlying data. * @param interval_ms Downsampling interval. */ - ValuesInInterval(final SeekableView source, final long interval_ms) { + ValuesInInterval(final SeekableView source, final long interval_ms, + final TimeZone timezone, boolean use_calendar) { this.source = source; this.interval_ms = interval_ms; this.timestamp_end_interval = interval_ms; + this.timezone = timezone != null ? timezone : TimeZone.getDefault(); + this.use_calendar = use_calendar; } /** Initializes to iterate intervals. */ @@ -177,8 +215,14 @@ private void moveToNextValue() { private void resetEndOfInterval() { if (has_next_value_from_source) { // Sets the end of the interval of the timestamp. - timestamp_end_interval = alignTimestamp(next_dp.timestamp()) + + + if (use_calendar && isCalendarInterval()) { + timestamp_end_interval = toEndOfInterval(next_dp.timestamp()); + } else { + // default timestamp normalization (tsdb v2.1.0) + timestamp_end_interval = alignTimestamp(next_dp.timestamp()) + interval_ms; + } } } @@ -193,8 +237,14 @@ void seekInterval(final long timestamp) { // To make sure that the interval of the given timestamp is fully filled, // rounds up the seeking timestamp to the smallest timestamp that is // a multiple of the interval and is greater than or equal to the given - // timestamp.. - source.seek(alignTimestamp(timestamp + interval_ms - 1)); + // timestamp. + if (use_calendar && isCalendarInterval()) { + source.seek(alignTimestamp(timestamp + toEndOfInterval(timestamp) + - toStartOfInterval(timestamp))); + } else { + source.seek(alignTimestamp(timestamp + interval_ms - 1)); + } + initialized = false; } @@ -203,13 +253,111 @@ protected long getIntervalTimestamp() { // NOTE: It is well-known practice taking the start time of // a downsample interval as a representative timestamp of it. It also // provides the correct context for seek. - return alignTimestamp(timestamp_end_interval - interval_ms); + if (use_calendar && isCalendarInterval()) { + return toStartOfInterval(timestamp_end_interval); + } else { + return alignTimestamp(timestamp_end_interval - interval_ms); + } } /** Returns timestamp aligned by interval. */ protected long alignTimestamp(final long timestamp) { - return timestamp - (timestamp % interval_ms); + if (use_calendar && isCalendarInterval()) { + return toStartOfInterval(timestamp); + } else { + return timestamp - (timestamp % interval_ms); + } + } + + /** Returns a flag denoting whether the interval can + * be aligned to the calendar */ + private boolean isCalendarInterval () { + if (interval_ms != 0 && + (interval_ms % ONE_YEAR_INTERVAL == 0 || + interval_ms % ONE_MONTH_INTERVAL == 0 || + interval_ms % ONE_WEEK_INTERVAL == 0 || + interval_ms % ONE_DAY_INTERVAL == 0)) { + return true; + } + return false; + } + + /** Returns a timestamp corresponding to the start of the interval + * in which the specified timestamp occurs, aligned to the calendar + * based on the timezone. */ + private long toStartOfInterval(long timestamp) { + if (interval_ms % ONE_YEAR_INTERVAL == 0) { + final long multiplier = interval_ms / ONE_YEAR_INTERVAL; + long result = timestamp; + for (long i = 0; i < multiplier; i++) { + result = toStartOfYear(result, timezone) - 1; + } + return result + 1; + } else if (interval_ms % ONE_MONTH_INTERVAL == 0) { + final long multiplier = interval_ms / ONE_MONTH_INTERVAL; + long result = timestamp; + for (long i = 0; i < multiplier; i++) { + result = toStartOfMonth(result, timezone) - 1; + } + return result + 1; + } else if (interval_ms % ONE_WEEK_INTERVAL == 0) { + final long multiplier = interval_ms / ONE_WEEK_INTERVAL; + long result = timestamp; + for (long i = 0; i < multiplier; i++) { + result = toStartOfWeek(result, timezone) - 1; + } + return result + 1; + } else if (interval_ms % ONE_DAY_INTERVAL == 0) { + final long multiplier = interval_ms / ONE_DAY_INTERVAL; + long result = timestamp; + for (long i = 0; i < multiplier; i++) { + result = toStartOfDay(result, timezone) - 1; + } + return result + 1; + } else { + throw new IllegalArgumentException(interval_ms + " does not correspond to a " + + "an interval that can be aligned to the calendar."); + } + } + + /** Returns a timestamp corresponding to the end of the interval + * in which the specified timestamp occurs, aligned to the calendar + * based on the timezone. */ + private long toEndOfInterval(long timestamp) { + if (interval_ms % ONE_YEAR_INTERVAL == 0) { + final long multiplier = interval_ms / ONE_YEAR_INTERVAL; + long result = timestamp; + for (long i = 0; i < multiplier; i++) { + result = toEndOfYear(result, timezone) + 1; + } + return result - 1; + } else if (interval_ms % ONE_MONTH_INTERVAL == 0) { + final long multiplier = interval_ms / ONE_MONTH_INTERVAL; + long result = timestamp; + for (long i = 0; i < multiplier; i++) { + result = toEndOfMonth(result, timezone) + 1; + } + return result - 1; + } else if (interval_ms % ONE_WEEK_INTERVAL == 0) { + final long multiplier = interval_ms / ONE_WEEK_INTERVAL; + long result = timestamp; + for (long i = 0; i < multiplier; i++) { + result = toEndOfWeek(result, timezone) + 1; + } + return result - 1; + } else if (interval_ms % ONE_DAY_INTERVAL == 0) { + final long multiplier = interval_ms / ONE_DAY_INTERVAL; + long result = timestamp; + for (long i = 0; i < multiplier; i++) { + result = toEndOfDay(result, timezone) + 1; + } + return result - 1; + } else { + throw new IllegalArgumentException(interval_ms + " does not correspond to a " + + "an interval that can be aligned to the calendar."); + } } + // ---------------------- // // Doubles interface // diff --git a/src/core/FillingDownsampler.java b/src/core/FillingDownsampler.java index 0e4d77ad10..6832f2b033 100644 --- a/src/core/FillingDownsampler.java +++ b/src/core/FillingDownsampler.java @@ -13,6 +13,7 @@ package net.opentsdb.core; import java.util.NoSuchElementException; +import java.util.TimeZone; /** * A specialized downsampler that returns special values, based on the fill @@ -30,7 +31,7 @@ public class FillingDownsampler extends Downsampler { protected final FillPolicy fill_policy; /** - * Create a new nulling downsampler. + * Ctor (preserved for backward compatibility). * @param source The iterator to access the underlying data. * @param start_time The time in milliseconds at which the data begins. * @param end_time The time in milliseconds at which the data ends. @@ -44,8 +45,30 @@ public class FillingDownsampler extends Downsampler { FillingDownsampler(final SeekableView source, final long start_time, final long end_time, final long interval_ms, final Aggregator downsampler, final FillPolicy fill_policy) { + this(source, start_time, end_time, interval_ms, downsampler, fill_policy, null, false); + } + + + /** + * Create a new nulling downsampler. + * @param source The iterator to access the underlying data. + * @param start_time The time in milliseconds at which the data begins. + * @param end_time The time in milliseconds at which the data ends. + * @param interval_ms The interval in milli seconds wanted between each data + * point. + * @param downsampler The downsampling function to use. + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @param timezone The timezone to use for aligning intervals based on the calendar. + * @throws IllegalArgumentException if fill_policy is interpolation. + */ + FillingDownsampler(final SeekableView source, final long start_time, + final long end_time, final long interval_ms, + final Aggregator downsampler, final FillPolicy fill_policy, + final TimeZone timezone, + final boolean use_calendar) { // Lean on the superclass implementation. - super(source, interval_ms, downsampler); + super(source, interval_ms, downsampler, timezone, use_calendar); // Ensure we aren't given a bogus fill policy. if (FillPolicy.NONE == fill_policy) { diff --git a/src/core/Query.java b/src/core/Query.java index 34553c591e..fb3a05d57a 100644 --- a/src/core/Query.java +++ b/src/core/Query.java @@ -66,7 +66,25 @@ public interface Query { * @return A strictly positive integer. */ long getEndTime(); - + + /** + * Sets the timezone to use for aligning intervals based on the calendar. + * @param timezone the timezone to use + */ + void setTimezone(String timezone); + + /** @return the timezone to use for aligning intervals based on the calendar. */ + String getTimezone(); + + /** + * Sets a flag denoting whether or not to align intervals based on the calendar. + * @param use_calendar true, if the intervals should be aligned based on the calendar; false, otherwise + */ + void setUseCalendar(boolean use_calendar); + + /** @return A flag denoting whether or not to align intervals based on the calendar. */ + boolean getUseCalendar(); + /** * Sets whether or not the data queried will be deleted. * @param delete True if data should be deleted, false otherwise. diff --git a/src/core/Span.java b/src/core/Span.java index c49ee773cb..c203fa3324 100644 --- a/src/core/Span.java +++ b/src/core/Span.java @@ -18,8 +18,10 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.TimeZone; import net.opentsdb.meta.Annotation; +import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.UniqueId; import org.hbase.async.Bytes; @@ -446,6 +448,19 @@ public String toString() { } + /** + * Package private iterator method to access data while downsampling. Preserved for backward + * compatibility. + * @param interval_ms The interval in milli seconds wanted between each data + * point. + * @param downsampler The downsampling function to use. + */ + Downsampler downsampler(final long interval_ms, + final Aggregator downsampler) { + return new Downsampler(spanIterator(), interval_ms, downsampler, null, false); + } + + /** * Package private iterator method to access data while downsampling with the * option to force interpolation. @@ -456,22 +471,26 @@ public String toString() { * @param downsampler The downsampling function to use. * @param fill_policy Policy specifying whether to interpolate or to fill * missing intervals with special values. + * @param timezone The timezone used for purposes of defining interval boundaries + * @param use_calendar A flag denoting whether or not to align intervals based on the calendar * @return A new downsampler. */ Downsampler downsampler(final long start_time, final long end_time, final long interval_ms, final Aggregator downsampler, - final FillPolicy fill_policy) { + final FillPolicy fill_policy, + final TimeZone timezone, + final boolean use_calendar) { if (FillPolicy.NONE == fill_policy) { // The default downsampler simply skips missing intervals, causing the // span group to linearly interpolate. - return new Downsampler(spanIterator(), interval_ms, downsampler); + return new Downsampler(spanIterator(), interval_ms, downsampler, timezone, use_calendar); } else { // Otherwise, we need to instantiate a downsampler that can fill missing // intervals with special values. return new FillingDownsampler(spanIterator(), start_time, end_time, - interval_ms, downsampler, fill_policy); + interval_ms, downsampler, fill_policy, timezone, use_calendar); } } diff --git a/src/core/SpanGroup.java b/src/core/SpanGroup.java index 026cb0fedc..fe1ade6f4b 100644 --- a/src/core/SpanGroup.java +++ b/src/core/SpanGroup.java @@ -98,7 +98,7 @@ final class SpanGroup implements DataPoints { /** Minimum time interval (in seconds) wanted between each data point. */ private final long sample_interval; - + /** Index of the query in the TSQuery class */ private final int query_index; @@ -108,21 +108,29 @@ final class SpanGroup implements DataPoints { /** The TSDB to which we belong, used for resolution */ private final TSDB tsdb; + /** The timezone to use for aligning intervals based on the calendar */ + private final String timezone; + + /** A flag denoting whether or not to align intervals based on the calendar */ + private final boolean use_calendar; + /** - * Ctor. + * Ctor (preserved for purposes of backward compatibility). * @param tsdb The TSDB we belong to. * @param start_time Any data point strictly before this timestamp will be * ignored. * @param end_time Any data point strictly after this timestamp will be * ignored. * @param spans A sequence of initial {@link Spans} to add to this group. - * Ignored if {@code null}. Additional spans can be added with {@link #add}. + * Ignored if {@code null}. Additional spans can be added with {@link #add}. * @param rate If {@code true}, the rate of the series will be used instead * of the actual values. + * @param rate_options Specifies the optional additional rate calculation options. * @param aggregator The aggregation function to use. * @param interval Number of milliseconds wanted between each data point. * @param downsampler Aggregation function to use to group data points * within an interval. + * @since 2.0 */ SpanGroup(final TSDB tsdb, final long start_time, final long end_time, @@ -131,10 +139,10 @@ final class SpanGroup implements DataPoints { final Aggregator aggregator, final long interval, final Aggregator downsampler) { this(tsdb, start_time, end_time, spans, rate, new RateOptions(false, - Long.MAX_VALUE, RateOptions.DEFAULT_RESET_VALUE), aggregator, interval, - downsampler); + Long.MAX_VALUE, RateOptions.DEFAULT_RESET_VALUE), aggregator, + interval, downsampler); } - + /** * Ctor. * @param tsdb The TSDB we belong to. @@ -151,6 +159,8 @@ final class SpanGroup implements DataPoints { * @param interval Number of milliseconds wanted between each data point. * @param downsampler Aggregation function to use to group data points * within an interval. + * @param timezone The timezone to use for aligning intervals based on the calendar. + * @param use_calendar A flag denoting whether or not to align intervals based on the calendar. * @since 2.0 */ SpanGroup(final TSDB tsdb, @@ -160,8 +170,8 @@ final class SpanGroup implements DataPoints { final Aggregator aggregator, final long interval, final Aggregator downsampler) { this(tsdb, start_time, end_time, spans, rate, rate_options, aggregator, - interval, downsampler, -1, FillPolicy.NONE); - } + interval, downsampler, -1, FillPolicy.NONE, null, false); + } /** * Ctor. @@ -191,6 +201,42 @@ final class SpanGroup implements DataPoints { final Aggregator aggregator, final long interval, final Aggregator downsampler, final int query_index, final FillPolicy fill_policy) { + this(tsdb, start_time, end_time, spans, rate, rate_options, aggregator, + interval, downsampler, query_index, fill_policy, null, false); + + } + + /** + * Ctor. + * @param tsdb The TSDB we belong to. + * @param start_time Any data point strictly before this timestamp will be + * ignored. + * @param end_time Any data point strictly after this timestamp will be + * ignored. + * @param spans A sequence of initial {@link Spans} to add to this group. + * Ignored if {@code null}. Additional spans can be added with {@link #add}. + * @param rate If {@code true}, the rate of the series will be used instead + * of the actual values. + * @param rate_options Specifies the optional additional rate calculation options. + * @param aggregator The aggregation function to use. + * @param interval Number of milliseconds wanted between each data point. + * @param downsampler Aggregation function to use to group data points + * within an interval. + * @param query_index index of the original query + * @param fill_policy Policy specifying whether to interpolate or to fill + * missing intervals with special values. + * @param timezone The timezone to use for aligning intervals based on the calendar. + * @since 2.2 + */ + SpanGroup(final TSDB tsdb, + final long start_time, final long end_time, + final Iterable spans, + final boolean rate, final RateOptions rate_options, + final Aggregator aggregator, + final long interval, final Aggregator downsampler, final int query_index, + final FillPolicy fill_policy, + final String timezone, + final boolean use_calendar) { annotations = new ArrayList(); this.start_time = (start_time & Const.SECOND_MASK) == 0 ? start_time * 1000 : start_time; @@ -209,6 +255,8 @@ final class SpanGroup implements DataPoints { this.query_index = query_index; this.fill_policy = fill_policy; this.tsdb = tsdb; + this.timezone = timezone; + this.use_calendar = use_calendar; } /** @@ -431,7 +479,8 @@ public SeekableView iterator() { return AggregationIterator.create(spans, start_time, end_time, aggregator, aggregator.interpolationMethod(), downsampler, sample_interval, - rate, rate_options, fill_policy); + rate, rate_options, fill_policy, timezone, + use_calendar); } /** diff --git a/src/core/TSQuery.java b/src/core/TSQuery.java index 5310d9a4cf..86e8cf7738 100644 --- a/src/core/TSQuery.java +++ b/src/core/TSQuery.java @@ -95,6 +95,9 @@ public final class TSQuery { /** The query status for tracking over all performance of this query */ private QueryStats query_stats; + + /** A flag denoting whether or not to align intervals based on the calendar */ + private boolean use_calendar; /** * Default constructor necessary for POJO de/serialization @@ -303,6 +306,11 @@ public String getTimezone() { return timezone; } + /** @return the flag denoting whether intervals should be aligned based on the calendar */ + public boolean getUseCalendar() { + return use_calendar; + } + /** @return a map of serializer options */ public Map> getOptions() { return options; @@ -388,6 +396,11 @@ public void setTimezone(String timezone) { this.timezone = timezone; } + /** @param use_calendar a flag denoting whether or not to align intervals based on the calendar */ + public void setUseCalendar(boolean use_calendar) { + this.use_calendar = use_calendar; + } + /** @param options a map of options to pass on to the serializer */ public void setOptions(HashMap> options) { this.options = options; diff --git a/src/core/TsdbQuery.java b/src/core/TsdbQuery.java index 7b060a44ed..d58e9f9842 100644 --- a/src/core/TsdbQuery.java +++ b/src/core/TsdbQuery.java @@ -24,28 +24,28 @@ import java.util.Set; import java.util.TreeMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import net.opentsdb.query.QueryUtil; +import net.opentsdb.query.filter.TagVFilter; +import net.opentsdb.stats.Histogram; +import net.opentsdb.uid.NoSuchUniqueId; +import net.opentsdb.uid.NoSuchUniqueName; +import net.opentsdb.uid.UniqueId; +import net.opentsdb.utils.DateTime; + import org.hbase.async.Bytes; +import org.hbase.async.Bytes.ByteMap; import org.hbase.async.DeleteRequest; import org.hbase.async.HBaseException; import org.hbase.async.KeyValue; import org.hbase.async.Scanner; -import org.hbase.async.Bytes.ByteMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; import com.stumbleupon.async.DeferredGroupException; -import net.opentsdb.query.QueryUtil; -import net.opentsdb.query.filter.TagVFilter; -import net.opentsdb.stats.Histogram; -import net.opentsdb.uid.NoSuchUniqueId; -import net.opentsdb.uid.NoSuchUniqueName; -import net.opentsdb.uid.UniqueId; -import net.opentsdb.utils.DateTime; - /** * Non-synchronized implementation of {@link Query}. */ @@ -131,6 +131,12 @@ final class TsdbQuery implements Query { /** Tag value filters to apply post scan */ private List filters; + + /** The timezone to use for aligning intervals based on the calendar */ + private String timezone; + + /** A flag denoting whether or not to align intervals based on the calendar */ + private boolean use_calendar; /** Constructor. */ public TsdbQuery(final TSDB tsdb) { @@ -200,6 +206,10 @@ public long getEndTime() { return end_time; } + /** + * Sets the timezone to use for aligning intervals based on the calendar. + * @param timezone the timezone to use + */ @Override public void setDelete(boolean delete) { this.delete = delete; @@ -210,6 +220,31 @@ public boolean getDelete() { return delete; } + public void setTimezone(String timezone) { + this.timezone = timezone; + } + + /** @return the timezone to use for aligning intervals based on the calendar. */ + @Override + public String getTimezone() { + return this.timezone; + } + + /** + * Sets a flag denoting whether or not to align intervals based on the calendar. + * @param use_calendar true, if the intervals should be aligned based on the calendar; false, otherwise + */ + @Override + public void setUseCalendar(boolean use_calendar) { + this.use_calendar = use_calendar; + } + + /** @return A flag denoting whether or not to align intervals based on the calendar. */ + @Override + public boolean getUseCalendar() { + return this.use_calendar; + } + @Override public void setTimeSeries(final String metric, final Map tags, @@ -311,6 +346,8 @@ public Deferred configureFromQuery(final TSQuery query, final TSSubQuery sub_query = query.getQueries().get(index); setStartTime(query.startTime()); setEndTime(query.endTime()); + setTimezone(query.getTimezone()); + setUseCalendar(query.getUseCalendar()); setDelete(query.getDelete()); query_index = index; @@ -774,7 +811,8 @@ public DataPoints[] call(final TreeMap spans) throws Exception { rate, rate_options, aggregator, sample_interval_ms, downsampler, - query_index, fill_policy); + query_index, fill_policy, timezone, + use_calendar); return new SpanGroup[] { group }; } @@ -819,7 +857,7 @@ public DataPoints[] call(final TreeMap spans) throws Exception { getScanEndTimeSeconds(), null, rate, rate_options, aggregator, sample_interval_ms, downsampler, query_index, - fill_policy); + fill_policy, timezone, use_calendar); // Copy the array because we're going to keep `group' and overwrite // its contents. So we want the collection to have an immutable copy. final byte[] group_copy = new byte[group.length]; diff --git a/src/tsd/QueryRpc.java b/src/tsd/QueryRpc.java index 58f1ae38fb..664636bb62 100644 --- a/src/tsd/QueryRpc.java +++ b/src/tsd/QueryRpc.java @@ -473,6 +473,23 @@ private TSQuery parseQuery(final TSDB tsdb, final HttpQuery query) { data_query.setShowSummary(true); } + if (query.hasQueryStringParam("tz")) { + data_query.setTimezone(query.getQueryStringParam("tz")); + } + + if (query.hasQueryStringParam("use_calendar")) { + final String val = query.getQueryStringParam("use_calendar").trim().toUpperCase(); + final boolean use_calendar; + if (val.equals("TRUE")) { + use_calendar = true; + } else { + use_calendar = false; + } + data_query.setUseCalendar(use_calendar); + } else { + data_query.setUseCalendar(tsdb.getConfig().use_calendar()); + } + // handle tsuid queries first if (query.hasQueryStringParam("tsuid")) { final List tsuids = query.getQueryStringParams("tsuid"); diff --git a/src/utils/Config.java b/src/utils/Config.java index 3344ff2e3c..6841a955f3 100644 --- a/src/utils/Config.java +++ b/src/utils/Config.java @@ -103,6 +103,9 @@ public class Config { /** tsd.core.tree.enable_processing */ private boolean enable_tree_processing = false; + /** tsd.query.downsample.use_calendar */ + private boolean use_calendar = false; + /** * The list of properties configured to their defaults or modified by users */ @@ -245,6 +248,12 @@ public boolean enable_tree_processing() { return enable_tree_processing; } + /** @return whether or not to align to the Gregorian calendar when downsampling a + * daily, weekly, monthly, or yearly interval */ + public boolean use_calendar() { + return use_calendar; + } + /** * Allows for modifying properties after creation or loading. * @@ -523,6 +532,7 @@ protected void setDefaults() { + "Content-Type, Accept, Origin, User-Agent, DNT, Cache-Control, " + "X-Mx-ReqToken, Keep-Alive, X-Requested-With, If-Modified-Since"); default_map.put("tsd.query.timeout", "0"); + default_map.put("tsd.query.downsample.use_calendar", "false"); for (Map.Entry entry : default_map.entrySet()) { if (!properties.containsKey(entry.getKey())) @@ -635,6 +645,7 @@ protected void loadStaticVariables() { } enable_tree_processing = this.getBoolean("tsd.core.tree.enable_processing"); fix_duplicates = this.getBoolean("tsd.storage.fix_duplicates"); + use_calendar = this.getBoolean("tsd.query.downsample.use_calendar"); } /** diff --git a/src/utils/DateTime.java b/src/utils/DateTime.java index 97b052c0bb..687630fcf8 100644 --- a/src/utils/DateTime.java +++ b/src/utils/DateTime.java @@ -14,6 +14,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Calendar; import java.util.HashMap; import java.util.TimeZone; @@ -267,5 +268,175 @@ public static void setDefaultTimezone(final String tzname) { public static long currentTimeMillis() { return System.currentTimeMillis(); } + + /** + * Returns a timestamp corresponding the beginning of the year in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the beginning of the year + * @return the epoch time corresponding to the beginning of the year + */ + public static long toStartOfYear(final long timestamp, final TimeZone time_zone) { + final Calendar c = getStartOfYear(timestamp, time_zone); + return c.getTimeInMillis(); + } + + /** + * Returns a timestamp corresponding the end of the year in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the end of the year + * @return the epoch time corresponding to the end of the year + */ + public static long toEndOfYear(final long timestamp, final TimeZone time_zone) { + final Calendar c = getStartOfYear(timestamp, time_zone); + c.add(Calendar.YEAR, 1); + return c.getTimeInMillis() - 1; + } + + /** + * Returns a timestamp corresponding the beginning of the month in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the beginning of the month + * @return the epoch time corresponding to the beginning of the month + */ + public static long toStartOfMonth(final long timestamp, final TimeZone time_zone) { + final Calendar c = getStartOfMonth(timestamp, time_zone); + return c.getTimeInMillis(); + } + + /** + * Returns a timestamp corresponding the end of the month in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the end of the month + * @return the epoch time corresponding to the end of the month + */ + public static long toEndOfMonth(final long timestamp, final TimeZone time_zone) { + final Calendar c = getStartOfMonth(timestamp, time_zone); + c.add(Calendar.MONTH, 1); + return c.getTimeInMillis() - 1; + } + + /** + * Returns a timestamp corresponding the beginning of the week in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the beginning of the week + * @return the epoch time corresponding to the beginning of the week + */ + public static long toStartOfWeek(final long timestamp, final TimeZone time_zone) { + final Calendar c = getStartOfWeek(timestamp, time_zone); + return c.getTimeInMillis(); + } + + /** + * Returns a timestamp corresponding the end of the week in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the end of the week + * @return the epoch time corresponding to the end of the week + */ + public static long toEndOfWeek(final long timestamp, final TimeZone time_zone) { + final Calendar c = getStartOfWeek(timestamp, time_zone); + c.add(Calendar.DATE, 7); + return c.getTimeInMillis() - 1; + } + /** + * Returns a timestamp corresponding the beginning of the day in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the beginning of the day + * @return the epoch time corresponding to the beginning of the day + */ + public static long toStartOfDay(final long timestamp, final TimeZone time_zone) { + final Calendar c = getStartOfDay(timestamp, time_zone); + return c.getTimeInMillis(); + } + + /** + * Returns a timestamp corresponding the end of the day in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the end of the day + * @return the epoch time corresponding to the end of the day + */ + public static long toEndOfDay(final long timestamp, final TimeZone time_zone) { + final Calendar c = getStartOfDay(timestamp, time_zone); + c.add(Calendar.DATE, 1); + return c.getTimeInMillis() - 1; + } + + /** + * Returns a Calendar object corresponding the beginning of the year in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the beginning of the year + * @return a Calendar object corresponding to the beginning of the year + */ + private static Calendar getStartOfYear(final long timestamp, final TimeZone time_zone) { + final Calendar c = Calendar.getInstance(time_zone); + c.setTimeInMillis(timestamp); + c.set(Calendar.DAY_OF_YEAR, 1); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + return c; + } + + /** + * Returns a Calendar object corresponding the beginning of the month in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the beginning of the month + * @return a Calendar object corresponding to the beginning of the month + */ + private static Calendar getStartOfMonth(final long timestamp, final TimeZone time_zone) { + final Calendar c = Calendar.getInstance(time_zone); + c.setTimeInMillis(timestamp); + c.set(Calendar.DAY_OF_MONTH, 1); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + return c; + } + + /** + * Returns a Calendar object corresponding the beginning of the week in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the beginning of the week + * @return a Calendar object corresponding to the beginning of the week + */ + private static Calendar getStartOfWeek(final long timestamp, final TimeZone time_zone) { + final Calendar c = Calendar.getInstance(time_zone); + c.setTimeInMillis(timestamp); + c.set(Calendar.DAY_OF_WEEK, 1); // 1-sun, 2-mon. + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + return c; + } + + /** + * Returns a Calendar object corresponding the beginning of the day in which the specified + * timestamp occurs. This operation is performed based on the specified time zone. + * @param timestamp the epoch time + * @param time_zone the time zone used to determine the beginning of the day + * @return a Calendar object corresponding to the beginning of the day + */ + private static Calendar getStartOfDay(final long timestamp, final TimeZone time_zone) { + final Calendar c = Calendar.getInstance(time_zone); + c.setTimeInMillis(timestamp); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + return c; + } } diff --git a/test/core/TestDownsampler.java b/test/core/TestDownsampler.java index 32e94c8d2a..ebeaec9228 100644 --- a/test/core/TestDownsampler.java +++ b/test/core/TestDownsampler.java @@ -13,6 +13,10 @@ package net.opentsdb.core; +import static net.opentsdb.core.Downsampler.ONE_DAY_INTERVAL; +import static net.opentsdb.core.Downsampler.ONE_MONTH_INTERVAL; +import static net.opentsdb.core.Downsampler.ONE_WEEK_INTERVAL; +import static net.opentsdb.core.Downsampler.ONE_YEAR_INTERVAL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -20,15 +24,17 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import java.util.Calendar; import java.util.List; - -import com.google.common.collect.Lists; +import java.util.TimeZone; import net.opentsdb.utils.DateTime; import org.junit.Before; import org.junit.Test; +import com.google.common.collect.Lists; + /** Tests {@link Downsampler}. */ public class TestDownsampler { @@ -54,6 +60,9 @@ public class TestDownsampler { (int)DateTime.parseDuration("10s"); private static final Aggregator AVG = Aggregators.get("avg"); private static final Aggregator SUM = Aggregators.get("sum"); + private static final TimeZone UTC_TIME_ZONE = DateTime.timezones.get("UTC"); + private static final TimeZone EST_TIME_ZONE = DateTime.timezones.get("EST"); + private SeekableView source; private Downsampler downsampler; @@ -161,6 +170,411 @@ public void testDownsampler_15seconds() { assertEquals(48, values.get(3), 0.0000001); assertEquals(BASE_TIME + 45000L, timestamps_in_millis.get(3).longValue()); } + + @Test + public void testDownsampler_1day() { + final DataPoint [] data_points = new DataPoint[4]; + long timestamp = DateTime.toStartOfDay(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfDay(timestamp, UTC_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_DAY_INTERVAL, SUM); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(2, values.size()); + timestamp = DateTime.toStartOfDay(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0, j = 0; i < values.size(); i++) { + assertEquals((1 << j++) + (1 << j++), values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfDay(timestamp, UTC_TIME_ZONE) + 1; + } + } + + @Test + public void testDownsampler_1day_timezone() { + final DataPoint [] data_points = new DataPoint[4]; + long timestamp = DateTime.toStartOfDay(BASE_TIME, UTC_TIME_ZONE) - EST_TIME_ZONE.getOffset(BASE_TIME); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfDay(timestamp, EST_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_DAY_INTERVAL, SUM, EST_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(2, values.size()); + timestamp = DateTime.toStartOfDay(BASE_TIME, UTC_TIME_ZONE) - EST_TIME_ZONE.getOffset(BASE_TIME); + for (int i = 0, j = 0; i < values.size(); i++) { + assertEquals((1 << j++) + (1 << j++), values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfDay(timestamp, EST_TIME_ZONE) + 1; + } + } + + + @Test + public void testDownsampler_1week() { + final DataPoint [] data_points = new DataPoint[4]; + long timestamp = DateTime.toStartOfWeek(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfWeek(timestamp, UTC_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_WEEK_INTERVAL, SUM, UTC_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(2, values.size()); + timestamp = DateTime.toStartOfWeek(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0, j = 0; i < values.size(); i++) { + assertEquals((1 << j++) + (1 << j++), values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfWeek(timestamp, UTC_TIME_ZONE) + 1; + } + } + + @Test + public void testDownsampler_1week_timezone() { + final DataPoint [] data_points = new DataPoint[4]; + long timestamp = DateTime.toStartOfWeek(BASE_TIME, UTC_TIME_ZONE) - EST_TIME_ZONE.getOffset(BASE_TIME); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfWeek(timestamp, EST_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_WEEK_INTERVAL, SUM, EST_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(2, values.size()); + timestamp = DateTime.toStartOfWeek(BASE_TIME, UTC_TIME_ZONE) - EST_TIME_ZONE.getOffset(BASE_TIME); + for (int i = 0, j = 0; i < values.size(); i++) { + assertEquals((1 << j++) + (1 << j++), values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfWeek(timestamp, EST_TIME_ZONE) + 1; + } + } + + @Test + public void testDownsampler_1month() { + final DataPoint [] data_points = new DataPoint[24]; + long timestamp = DateTime.toStartOfMonth(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfMonth(timestamp, UTC_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_MONTH_INTERVAL, SUM, UTC_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(12, values.size()); + timestamp = DateTime.toStartOfMonth(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0, j = 0; i < values.size(); i++) { + assertEquals((1 << j++) + (1 << j++), values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfMonth(timestamp, UTC_TIME_ZONE) + 1; + } + } + + @Test + public void testDownsampler_1month_alt() { + /* + 1380600000 -> 2013-10-01T04:00:00Z + 1383278400 -> 2013-11-01T04:00:00Z + 1385874000 -> 2013-12-01T05:00:00Z + 1388552400 -> 2014-01-01T05:00:00Z + 1391230800 -> 2014-02-01T05:00:00Z + 1393650000 -> 2014-03-01T05:00:00Z + 1396324800 -> 2014-04-01T04:00:00Z + 1398916800 -> 2014-05-01T04:00:00Z + 1401595200 -> 2014-06-01T04:00:00Z + 1404187200 -> 2014-07-01T04:00:00Z + 1406865600 -> 2014-08-01T04:00:00Z + 1409544000 -> 2014-09-01T04:00:00Z + */ + + int value = 1; + final DataPoint [] data_points = new DataPoint[] { + MutableDataPoint.ofLongValue(1380600000000L, value), + MutableDataPoint.ofLongValue(1383278400000L, value), + MutableDataPoint.ofLongValue(1385874000000L, value), + MutableDataPoint.ofLongValue(1388552400000L, value), + MutableDataPoint.ofLongValue(1391230800000L, value), + MutableDataPoint.ofLongValue(1393650000000L, value), + MutableDataPoint.ofLongValue(1396324800000L, value), + MutableDataPoint.ofLongValue(1398916800000L, value), + MutableDataPoint.ofLongValue(1401595200000L, value), + MutableDataPoint.ofLongValue(1404187200000L, value), + MutableDataPoint.ofLongValue(1406865600000L, value), + MutableDataPoint.ofLongValue(1409544000000L, value), + }; + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_MONTH_INTERVAL, SUM, UTC_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(12, values.size()); + long timestamp = DateTime.toStartOfMonth(data_points[0].timestamp(), UTC_TIME_ZONE); + for (int i = 0; i < values.size(); i++) { + assertEquals(1, values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfMonth(timestamp, UTC_TIME_ZONE) + 1; + } + } + + + @Test + public void testDownsampler_2months() { + final DataPoint [] data_points = new DataPoint[24]; + long timestamp = DateTime.toStartOfMonth(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfMonth(timestamp, UTC_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, 2 * ONE_MONTH_INTERVAL, SUM, UTC_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(6, values.size()); + timestamp = DateTime.toStartOfMonth(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0, j = 0; i < values.size(); i++) { + long value = 0; + for (int k = 0; k < 4; k++) { + value += (1 << j++); + } + assertEquals(value, values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfMonth(timestamp, UTC_TIME_ZONE) + 1; + timestamp = DateTime.toEndOfMonth(timestamp, UTC_TIME_ZONE) + 1; + } + } + + @Test + public void testDownsampler_1month_timezone() { + final DataPoint [] data_points = new DataPoint[24]; + long timestamp = DateTime.toStartOfMonth(BASE_TIME, UTC_TIME_ZONE) - EST_TIME_ZONE.getOffset(BASE_TIME); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfMonth(timestamp, EST_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_MONTH_INTERVAL, SUM, EST_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(12, values.size()); + timestamp = DateTime.toStartOfMonth(BASE_TIME, UTC_TIME_ZONE) - EST_TIME_ZONE.getOffset(BASE_TIME); + for (int i = 0, j = 0; i < values.size(); i++) { + assertEquals((1 << j++) + (1 << j++), values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfMonth(timestamp, EST_TIME_ZONE) + 1; + } + } + + @Test + public void testDownsampler_1year() { + final DataPoint [] data_points = new DataPoint[4]; + long timestamp = DateTime.toStartOfYear(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfYear(timestamp, UTC_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_YEAR_INTERVAL, SUM, UTC_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(2, values.size()); + timestamp = DateTime.toStartOfYear(BASE_TIME, UTC_TIME_ZONE); + for (int i = 0, j = 0; i < values.size(); i++) { + assertEquals((1 << j++) + (1 << j++), values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfYear(timestamp, UTC_TIME_ZONE) + 1; + } + } + + @Test + public void testDownsampler_1year_timezone() { + final DataPoint [] data_points = new DataPoint[4]; + long timestamp = DateTime.toStartOfYear(BASE_TIME, UTC_TIME_ZONE) - EST_TIME_ZONE.getOffset(BASE_TIME); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + i += 1; + long startOfNextInterval = DateTime.toEndOfYear(timestamp, EST_TIME_ZONE) + 1; + timestamp = timestamp + (startOfNextInterval - timestamp) / 2; + value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + timestamp = startOfNextInterval; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + downsampler = new Downsampler(source, ONE_YEAR_INTERVAL, SUM, EST_TIME_ZONE, true); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(2, values.size()); + timestamp = DateTime.toStartOfYear(BASE_TIME, UTC_TIME_ZONE) - EST_TIME_ZONE.getOffset(BASE_TIME); + for (int i = 0, j = 0; i < values.size(); i++) { + assertEquals((1 << j++) + (1 << j++), values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfYear(timestamp, EST_TIME_ZONE) + 1; + } + } + @Test(expected = UnsupportedOperationException.class) public void testRemove() { @@ -180,7 +594,7 @@ public void testSeek() { values.add(dp.doubleValue()); timestamps_in_millis.add(dp.timestamp()); } - + assertEquals(3, values.size()); assertEquals(45, values.get(0), 0.0000001); assertEquals(BASE_TIME + 3600000L, timestamps_in_millis.get(0).longValue()); @@ -190,6 +604,69 @@ public void testSeek() { assertEquals(BASE_TIME + 8600000L, timestamps_in_millis.get(2).longValue()); } + @Test + public void testSeek_useCalendar() { + final DataPoint [] data_points = new DataPoint[4]; + long timestamp = DateTime.toStartOfYear(BASE_TIME, UTC_TIME_ZONE); + final Calendar c = Calendar.getInstance(UTC_TIME_ZONE); + c.setTimeInMillis(timestamp); + for (int i = 0; i < data_points.length; i++) { + long value = 1 << i; + data_points[i] = MutableDataPoint.ofLongValue(timestamp, value); + + timestamp = DateTime.toEndOfYear(timestamp, UTC_TIME_ZONE) + 1; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + c.add(Calendar.YEAR, 2); + downsampler = new Downsampler(source, ONE_YEAR_INTERVAL, SUM, UTC_TIME_ZONE, true); + downsampler.seek(c.getTimeInMillis()); + verify(source, never()).next(); + List values = Lists.newArrayList(); + List timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(2, values.size()); + timestamp = DateTime.toStartOfYear(c.getTimeInMillis(), UTC_TIME_ZONE); + for (int i = 2; i < values.size(); i++) { + System.out.println(timestamps_in_millis.get(i).longValue()); + assertEquals(1 << i, values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfYear(timestamp, UTC_TIME_ZONE) + 1; + } + + source = spy(SeekableViewsForTest.fromArray(data_points)); + + c.add(Calendar.MILLISECOND, 1); + downsampler = new Downsampler(source, ONE_YEAR_INTERVAL, SUM, UTC_TIME_ZONE, true); + downsampler.seek(c.getTimeInMillis()); + verify(source, never()).next(); + values = Lists.newArrayList(); + timestamps_in_millis = Lists.newArrayList(); + while (downsampler.hasNext()) { + DataPoint dp = downsampler.next(); + assertFalse(dp.isInteger()); + values.add(dp.doubleValue()); + timestamps_in_millis.add(dp.timestamp()); + } + + assertEquals(1, values.size()); + timestamp = DateTime.toStartOfYear(c.getTimeInMillis(), UTC_TIME_ZONE); + for (int i = 3; i < values.size(); i++) { + System.out.println(timestamps_in_millis.get(i).longValue()); + assertEquals(1 << i, values.get(i), 0.0000001); + assertEquals(timestamp, timestamps_in_millis.get(i).longValue()); + timestamp = DateTime.toEndOfYear(timestamp, UTC_TIME_ZONE) + 1; + } + + } + @Test public void testSeek_skipPartialInterval() { downsampler = new Downsampler(source, THOUSAND_SEC_INTERVAL, AVG); diff --git a/test/core/TestSpan.java b/test/core/TestSpan.java index dbfc247ce5..0825f562f4 100644 --- a/test/core/TestSpan.java +++ b/test/core/TestSpan.java @@ -18,10 +18,12 @@ import static org.powermock.api.mockito.PowerMockito.mock; import java.util.List; +import java.util.TimeZone; import net.opentsdb.storage.MockBase; import net.opentsdb.uid.UniqueId; import net.opentsdb.utils.Config; +import net.opentsdb.utils.DateTime; import org.hbase.async.Bytes; import org.hbase.async.KeyValue; @@ -58,6 +60,7 @@ public final class TestSpan { { 0, 0, 1, 0x50, (byte)0xE2, 0x43, 0x20, 0, 0, 1, 0, 0, 2 }; private static final byte[] FAMILY = { 't' }; private static final byte[] ZERO = { 0 }; + private static final TimeZone UTC_TIME_ZONE = DateTime.timezones.get("UTC"); @Before public void before() throws Exception { @@ -358,7 +361,7 @@ public void downsampler() throws Exception { long interval_ms = 1000000; Aggregator downsampler = Aggregators.get("avg"); final SeekableView it = span.downsampler(1356998000L, 1357007000L, - interval_ms, downsampler, FillPolicy.NONE); + interval_ms, downsampler, FillPolicy.NONE, UTC_TIME_ZONE, false); List values = Lists.newArrayList(); List timestamps_in_millis = Lists.newArrayList(); while (it.hasNext()) { diff --git a/test/tsd/TestQueryRpc.java b/test/tsd/TestQueryRpc.java index d498b963ff..2f041a5d8e 100644 --- a/test/tsd/TestQueryRpc.java +++ b/test/tsd/TestQueryRpc.java @@ -33,6 +33,7 @@ import net.opentsdb.query.filter.TagVRegexFilter; import net.opentsdb.query.filter.TagVWildcardFilter; import net.opentsdb.storage.MockDataPoints; +import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.utils.Config; import net.opentsdb.utils.DateTime; @@ -43,8 +44,6 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import net.opentsdb.uid.NoSuchUniqueName; - import com.stumbleupon.async.Deferred; import com.stumbleupon.async.DeferredGroupException; @@ -303,6 +302,59 @@ public void parseQueryMTypeWEmptyFilterBrackets() throws Exception { assertEquals(0, sub.getFilters().size()); } + @Test + public void parseQueryMTypeWTimeZone() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0&tz=EST"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + assertEquals("EST", tsq.getTimezone()); + } + + @Test + public void parseQueryMTypeWUseCalendar() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0&use_calendar=true"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + assertEquals(true, tsq.getUseCalendar()); + + /* If the tsdb.query.downsample.use_calendar Config setting is set to true, + * then it should be possible to override this setting through the + * use_calendar querystring parameter. + */ + TSDB tsdb2 = mock(TSDB.class); + Config config = mock(Config.class); + when(tsdb2.getConfig()).thenReturn(config); + when(config.use_calendar()).thenReturn(true); + + query = NettyMocks.getQuery(tsdb2, + "/api/query?start=1h-ago&m=sum:sys.cpu.0&use_calendar=false"); + tsq = (TSQuery) parseQuery.invoke(rpc, tsdb2, query); + assertEquals(false, tsq.getUseCalendar()); + + } + + @Test + public void parseQueryMTypeWOutUseCalendar() throws Exception { + HttpQuery query = NettyMocks.getQuery(tsdb, + "/api/query?start=1h-ago&m=sum:sys.cpu.0"); + TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query); + assertEquals(false, tsq.getUseCalendar()); + + /* If the tsdb.query.downsample.use_calendar Config setting is set to true, + * then the TSQuery objects use_calendar setting should also be set to + * true, even if the use_calendar parameter is missing from the query string + */ + TSDB tsdb2 = mock(TSDB.class); + Config config = mock(Config.class); + when(tsdb2.getConfig()).thenReturn(config); + when(config.use_calendar()).thenReturn(true); + + query = NettyMocks.getQuery(tsdb2, + "/api/query?start=1h-ago&m=sum:sys.cpu.0"); + tsq = (TSQuery) parseQuery.invoke(rpc, tsdb2, query); + assertEquals(true, tsq.getUseCalendar()); + } + @Test public void parseQueryTSUIDType() throws Exception { HttpQuery query = NettyMocks.getQuery(tsdb, diff --git a/test/utils/TestDateTime.java b/test/utils/TestDateTime.java index 23867039c4..198dec0b94 100644 --- a/test/utils/TestDateTime.java +++ b/test/utils/TestDateTime.java @@ -33,6 +33,8 @@ @PrepareForTest({ DateTime.class, System.class }) public final class TestDateTime { + private static final TimeZone UTC_TIME_ZONE = DateTime.timezones.get("UTC"); + @Before public void before() { PowerMockito.mockStatic(System.class); @@ -372,4 +374,48 @@ public void currentTimeMillis() { assertEquals(1388534400000L, DateTime.currentTimeMillis()); } + @Test + public void toStartOfMonthBoundaryChecks() { + // test start and end with timestamp corresponding to 2013-10-01T00:00:00Z + assertEquals(1380585600000L, DateTime.toStartOfMonth(1380585600000L, UTC_TIME_ZONE)); + assertEquals(1383263999999L, DateTime.toEndOfMonth(1380585600000L, UTC_TIME_ZONE)); + + // test start and end with timestamp corresponding to 2013-10-31T23:59:59Z + assertEquals(1380585600000L, DateTime.toStartOfMonth(1383263999999L, UTC_TIME_ZONE)); + assertEquals(1383263999999L, DateTime.toEndOfMonth(1383263999999L, UTC_TIME_ZONE)); + } + + @Test + public void toStartOfWeekBoundaryChecks() { + // test start and end with timestamp corresponding to 2015-12-061T00:00:00Z, which is a Sunday + assertEquals(1449360000000L, DateTime.toStartOfWeek(1449360000000L, UTC_TIME_ZONE)); + assertEquals(1449964799999L, DateTime.toEndOfWeek(1449360000000L, UTC_TIME_ZONE)); + + // test start and end with timestamp corresponding to 2015-12-12T23:59:59Z + assertEquals(1449360000000L, DateTime.toStartOfWeek(1449964799999L, UTC_TIME_ZONE)); + assertEquals(1449964799999L, DateTime.toEndOfWeek(1449964799999L, UTC_TIME_ZONE)); + } + + @Test + public void toStartOfYearBoundaryChecks() { + // test start and end with timestamp corresponding to 2015-01-01T00:00:00Z + assertEquals(1420070400000L, DateTime.toStartOfYear(1420070400000L, UTC_TIME_ZONE)); + assertEquals(1451606399999L, DateTime.toEndOfYear(1420070400000L, UTC_TIME_ZONE)); + + // test start and end with timestamp corresponding to 2015-12-31T23:59:59Z + assertEquals(1420070400000L, DateTime.toStartOfYear(1451606399999L, UTC_TIME_ZONE)); + assertEquals(1451606399999L, DateTime.toEndOfYear(1451606399999L, UTC_TIME_ZONE)); + } + + @Test + public void toStartOfDayBoundaryChecks() { + // test start and end with timestamp corresponding to 2015-01-01T00:00:00Z + assertEquals(1420070400000L, DateTime.toStartOfDay(1420070400000L, UTC_TIME_ZONE)); + assertEquals(1420156799999L, DateTime.toEndOfDay(1420070400000L, UTC_TIME_ZONE)); + + // test start and end with timestamp corresponding to 2015-01-01T23:59:59Z + assertEquals(1420070400000L, DateTime.toStartOfDay(1420156799999L, UTC_TIME_ZONE)); + assertEquals(1420156799999L, DateTime.toEndOfDay(1420156799999L, UTC_TIME_ZONE)); + } + }