From 301641dd129e9556f146b760e6c261bc9155d12a Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 26 Nov 2024 11:08:28 +0530 Subject: [PATCH 01/21] Pre-commit fixes --- .../metrics/anomaly_detection/ts_precision.py | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 aeon/benchmarking/metrics/anomaly_detection/ts_precision.py diff --git a/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py b/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py new file mode 100644 index 0000000000..00071066ad --- /dev/null +++ b/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py @@ -0,0 +1,215 @@ +"""Calculate the precision of a time series anomaly detection model.""" + + +class RangePrecision: + """ + Calculates Precision for time series. + + Parameters + ---------- + y_real : np.ndarray + set of ground truth anomaly ranges (actual anomalies). + + y_pred : np.ndarray + set of predicted anomaly ranges. + + cardinality : str, default="one" + Number of overlaps between y_pred and y_real. + + gamma : float, default=1.0 + Overlpa Cardinality Factor. Penalizes or adjusts the metric based on + the cardinality. + Should be one of {'reciprocal', 'one', 'udf_gamma'}. + + alpha : float + Weight of the existence reward. Because precision by definition emphasizes on + prediction quality, there is no need for an existence reward and this value + should always be set to 0. + + bias : str, default="flat" + Captures importance of positional factors within anomaly ranges. + Determines the weight given to specific portions of anomaly range + when calculating overlap rewards. + Should be one of {'flat', 'front', 'middle', 'back'}. + + 'flat' - All positions are equally important. + 'front' - Front positions are more important. + 'middle' - Middle positions are more important. + 'back' - Back positions are more important. + + omega : float + Measure the extent and overlap between y_pred and y_real. + Considers the size and position of overlap and rewards. Should + be a float value between 0 and 1. + """ + + def __init__(self, bias="flat", alpha=0.0, gamma=None): + assert gamma in ["reciprocal", "one", "udf_gamma"], "Invalid gamma type" + assert bias in ["flat", "front", "middle", "back"], "Invalid bias type" + + self.bias = bias + self.alpha = alpha + self.gamma = gamma + + def calculate_overlap_set(y_pred, y_real): + """ + Calculate the overlap set for all predicted and real ranges. + + Parameters + ---------- + y_pred : np.ndarray + + y_real : np.ndarray + + Returns + ------- + list of sets : List where each set represents the 'overlap positions' + for a predicted range. + + Example + ------- + y_pred = [(1, 5), (10, 15)] + y_real = [(3, 8), (4, 6), (12, 18)] + + Output -> [ {3, 4, 5}, {12, 13, 14, 15} ] + """ + overlap_sets = [] + + for pred_start, pred_end in y_pred: + overlap_set = set() + for real_start, real_end in y_real: + overlap_start = max(pred_start, real_start) + overlap_end = min(pred_end, real_end) + + if overlap_start <= overlap_end: + overlap_set.update( + range(overlap_start, overlap_end + 1) + ) # Update set with overlap positions + overlap_sets.append(overlap_set) + + return overlap_sets + + def calculate_bias(i, length, bias_type="flat"): + """Calculate the bias value for a given postion in a range. + + Parameters + ---------- + i : int + Position index within the range. + + length : int + Total length of the range. + """ + if bias_type == "flat": + return 1 + elif bias_type == "front": + return length - i - 1 + elif bias_type == "back": + return i + elif bias_type == "middle": + if i <= length / 2: + return i + else: + return length - i - 1 + else: + raise ValueError( + f"Invalid bias type: {bias_type}." + "Should be from 'flat, 'front', 'middle', 'back'." + ) + + def gamma_select(self, cardinality, gamma: str, udf_gamma=None) -> float: + """Select a gamma value based on the cardinality type.""" + if gamma == "one": + return 1.0 + elif gamma == "reciprocal": + if cardinality > 1: + return 1 / cardinality + else: + return 1.0 + elif gamma == "udf_gamma": + if udf_gamma is not None: + return 1.0 / udf_gamma + else: + raise ValueError( + "udf_gamma must be provided for 'udf_gamma' gamma type." + ) + else: + raise ValueError("Invalid gamma type") + + def calculate_overlap_reward(self, y_pred, overlap_set, bias_type): + """Overlap Reward for y_pred.""" + start, end = y_pred + length = end - start + 1 + + max_value = 0 # Total possible weighted value for all positions. + my_value = 0 # Weighted value for overlapping positions only. + + for i in range(1, length + 1): + global_position = start + i - 1 + bias_value = self.calculate_bias(i, length, bias_type) + max_value += bias_value + + if global_position in overlap_set: + my_value += bias_value + + return my_value / max_value if max_value > 0 else 0.0 + + def ts_precision( + self, y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None + ): + """Precision for either a single set or the entire time series.""" + # Check if the input is a single set of predicted ranges or multiple sets + is_single_set = isinstance(y_pred[0], tuple) + """ + example: + y_pred = [(1, 3), (5, 7)] + y_real = [(2, 6), (8, 10)] + """ + if is_single_set: + # y_pred is a single set of predicted ranges + total_overlap_reward = 0.0 + total_cardinality = 0 + + for pred_range in y_pred: + overlap_set = set() + for real_start, real_end in y_real: + overlap_set.update( + range( + max(pred_range[0], real_start), + min(pred_range[1], real_end) + 1, + ) + ) + + overlap_reward = self.calculate_overlap_reward( + y_pred, overlap_set, bias_type + ) + cardinality = len(overlap_set) + gamma_value = self.gamma_select(cardinality, gamma, udf_gamma) + + total_overlap_reward += gamma_value * overlap_reward + total_cardinality += 1 # Count each predicted range once + + return ( + total_overlap_reward / total_cardinality + if total_cardinality > 0 + else 0.0 + ) + + else: + """ + example: + y_pred = [[(1, 3), (5, 7)],[(10, 12)]] + y_real = [(2, 6), (8, 10)] + """ + # y_pred as multiple sets of predicted ranges + total_precision = 0.0 + total_ranges = 0 + + for pred_ranges in y_pred: # Iterate over all sets of predicted ranges + precision = self.ts_precision( + pred_ranges, y_real, gamma, bias_type, udf_gamma + ) # Recursive call for single sets + total_precision += precision + total_ranges += len(pred_ranges) + + return total_precision / total_ranges if total_ranges > 0 else 0.0 From cc1101ac10a97984c28138308ba35ab643240bc7 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 26 Nov 2024 15:13:28 +0530 Subject: [PATCH 02/21] Position parameter in calculate_bias --- .../metrics/anomaly_detection/ts_precision.py | 71 +++++++++---------- 1 file changed, 32 insertions(+), 39 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py b/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py index 00071066ad..9730938992 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py +++ b/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py @@ -89,33 +89,26 @@ def calculate_overlap_set(y_pred, y_real): return overlap_sets - def calculate_bias(i, length, bias_type="flat"): - """Calculate the bias value for a given postion in a range. + def calculate_bias(self, position, length, bias_type="flat"): + """Calculate bias value based on position and length. - Parameters - ---------- - i : int - Position index within the range. - - length : int - Total length of the range. + Args: + position: Current position in the range + length: Total length of the range + bias_type: Type of bias to apply (default: "flat") """ if bias_type == "flat": - return 1 + return 1.0 elif bias_type == "front": - return length - i - 1 - elif bias_type == "back": - return i + return 1.0 - (position - 1) / length elif bias_type == "middle": - if i <= length / 2: - return i - else: - return length - i - 1 - else: - raise ValueError( - f"Invalid bias type: {bias_type}." - "Should be from 'flat, 'front', 'middle', 'back'." + return ( + 1.0 - abs(2 * (position - 1) / (length - 1) - 1) if length > 1 else 1.0 ) + elif bias_type == "back": + return position / length + else: + return 1.0 def gamma_select(self, cardinality, gamma: str, udf_gamma=None) -> float: """Select a gamma value based on the cardinality type.""" @@ -136,9 +129,9 @@ def gamma_select(self, cardinality, gamma: str, udf_gamma=None) -> float: else: raise ValueError("Invalid gamma type") - def calculate_overlap_reward(self, y_pred, overlap_set, bias_type): + def calculate_overlap_reward(self, pred_range, overlap_set, bias_type): """Overlap Reward for y_pred.""" - start, end = y_pred + start, end = pred_range length = end - start + 1 max_value = 0 # Total possible weighted value for all positions. @@ -157,37 +150,37 @@ def calculate_overlap_reward(self, y_pred, overlap_set, bias_type): def ts_precision( self, y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None ): - """Precision for either a single set or the entire time series.""" - # Check if the input is a single set of predicted ranges or multiple sets - is_single_set = isinstance(y_pred[0], tuple) - """ + """Precision for either a single set or the entire time series. + example: y_pred = [(1, 3), (5, 7)] y_real = [(2, 6), (8, 10)] """ - if is_single_set: + # Check if the input is a single set of predicted ranges or multiple sets + if isinstance(y_pred[0], tuple): # y_pred is a single set of predicted ranges total_overlap_reward = 0.0 total_cardinality = 0 for pred_range in y_pred: overlap_set = set() + cardinality = 0 + for real_start, real_end in y_real: - overlap_set.update( - range( - max(pred_range[0], real_start), - min(pred_range[1], real_end) + 1, - ) - ) + overlap_start = max(pred_range[0], real_start) + overlap_end = min(pred_range[1], real_end) + + if overlap_start <= overlap_end: + overlap_set.update(range(overlap_start, overlap_end + 1)) + cardinality += 1 overlap_reward = self.calculate_overlap_reward( - y_pred, overlap_set, bias_type + pred_range, overlap_set, bias_type ) - cardinality = len(overlap_set) gamma_value = self.gamma_select(cardinality, gamma, udf_gamma) - total_overlap_reward += gamma_value * overlap_reward - total_cardinality += 1 # Count each predicted range once + total_overlap_reward += gamma_value * overlap_reward + total_cardinality += 1 return ( total_overlap_reward / total_cardinality @@ -209,7 +202,7 @@ def ts_precision( precision = self.ts_precision( pred_ranges, y_real, gamma, bias_type, udf_gamma ) # Recursive call for single sets - total_precision += precision + total_precision += precision * len(pred_ranges) total_ranges += len(pred_ranges) return total_precision / total_ranges if total_ranges > 0 else 0.0 From 10289423a278cd207500dcaade6ab685bd617e81 Mon Sep 17 00:00:00 2001 From: Aryan Date: Sat, 30 Nov 2024 15:48:45 +0530 Subject: [PATCH 03/21] Added recall metric --- .../metrics/anomaly_detection/ts_precision.py | 42 +---- .../metrics/anomaly_detection/ts_recall.py | 148 ++++++++++++++++++ 2 files changed, 150 insertions(+), 40 deletions(-) create mode 100644 aeon/benchmarking/metrics/anomaly_detection/ts_recall.py diff --git a/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py b/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py index 9730938992..14cb519f7a 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py +++ b/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py @@ -51,44 +51,6 @@ def __init__(self, bias="flat", alpha=0.0, gamma=None): self.alpha = alpha self.gamma = gamma - def calculate_overlap_set(y_pred, y_real): - """ - Calculate the overlap set for all predicted and real ranges. - - Parameters - ---------- - y_pred : np.ndarray - - y_real : np.ndarray - - Returns - ------- - list of sets : List where each set represents the 'overlap positions' - for a predicted range. - - Example - ------- - y_pred = [(1, 5), (10, 15)] - y_real = [(3, 8), (4, 6), (12, 18)] - - Output -> [ {3, 4, 5}, {12, 13, 14, 15} ] - """ - overlap_sets = [] - - for pred_start, pred_end in y_pred: - overlap_set = set() - for real_start, real_end in y_real: - overlap_start = max(pred_start, real_start) - overlap_end = min(pred_end, real_end) - - if overlap_start <= overlap_end: - overlap_set.update( - range(overlap_start, overlap_end + 1) - ) # Update set with overlap positions - overlap_sets.append(overlap_set) - - return overlap_sets - def calculate_bias(self, position, length, bias_type="flat"): """Calculate bias value based on position and length. @@ -129,7 +91,7 @@ def gamma_select(self, cardinality, gamma: str, udf_gamma=None) -> float: else: raise ValueError("Invalid gamma type") - def calculate_overlap_reward(self, pred_range, overlap_set, bias_type): + def calculate_overlap_reward_precision(self, pred_range, overlap_set, bias_type): """Overlap Reward for y_pred.""" start, end = pred_range length = end - start + 1 @@ -174,7 +136,7 @@ def ts_precision( overlap_set.update(range(overlap_start, overlap_end + 1)) cardinality += 1 - overlap_reward = self.calculate_overlap_reward( + overlap_reward = self.calculate_overlap_reward_precision( pred_range, overlap_set, bias_type ) gamma_value = self.gamma_select(cardinality, gamma, udf_gamma) diff --git a/aeon/benchmarking/metrics/anomaly_detection/ts_recall.py b/aeon/benchmarking/metrics/anomaly_detection/ts_recall.py new file mode 100644 index 0000000000..6d204c49f4 --- /dev/null +++ b/aeon/benchmarking/metrics/anomaly_detection/ts_recall.py @@ -0,0 +1,148 @@ +"""Calculate the Recall metric of a time series anomaly detection model.""" + + +class RangeRecall: + """Calculates Recall for time series. + + Parameters + ---------- + y_real : np.ndarray + Set of ground truth anomaly ranges (actual anomalies). + + y_pred : np.ndarray + Set of predicted anomaly ranges. + + cardinality : str, default="one" + Number of overlaps between y_pred and y_real. + + gamma : float, default=1.0 + Overlap Cardinality Factor. Penalizes or adjusts the metric based on + the cardinality. + Should be one of {'reciprocal', 'one', 'udf_gamma'}. + + alpha : float + Weight of the existence reward. Since Recall emphasizes coverage, + you might adjust this value if needed. + + bias : str, default="flat" + Captures the importance of positional factors within anomaly ranges. + Determines the weight given to specific portions of anomaly range + when calculating overlap rewards. + Should be one of {'flat', 'front', 'middle', 'back'}. + + omega : float + Measure the extent and overlap between y_pred and y_real. + Considers the size and position of overlap and rewards. Should + be a float value between 0 and 1. + """ + + def _init_(self, bias="flat", alpha=0.0, gamma="one"): + assert gamma in ["reciprocal", "one", "udf_gamma"], "Invalid gamma type" + assert bias in ["flat", "front", "middle", "back"], "Invalid bias type" + + self.bias = bias + self.alpha = alpha + self.gamma = gamma + + def calculate_bias(self, position, length, bias_type="flat"): + """Calculate bias value based on position and length. + + Args: + position: Current position in the range + length: Total length of the range + bias_type: Type of bias to apply (default: "flat") + """ + if bias_type == "flat": + return 1.0 + elif bias_type == "front": + return 1.0 - (position - 1) / length + elif bias_type == "middle": + return ( + 1.0 - abs(2 * (position - 1) / (length - 1) - 1) if length > 1 else 1.0 + ) + elif bias_type == "back": + return position / length + else: + return 1.0 + + def gamma_select(self, cardinality, gamma: str, udf_gamma=None) -> float: + """Select a gamma value based on the cardinality type.""" + if gamma == "one": + return 1.0 + elif gamma == "reciprocal": + if cardinality > 1: + return 1 / cardinality + else: + return 1.0 + elif gamma == "udf_gamma": + if udf_gamma is not None: + return 1.0 / udf_gamma + else: + raise ValueError( + "udf_gamma must be provided for 'udf_gamma' gamma type." + ) + else: + raise ValueError("Invalid gamma type") + + def calculate_overlap_reward_recall(self, real_range, overlap_set, bias_type): + """Overlap Reward for y_real.""" + start, end = real_range + length = end - start + 1 + + max_value = 0.0 # Total possible weighted value for all positions. + my_value = 0.0 # Weighted value for overlapping positions only. + + for i in range(1, length + 1): + global_position = start + i - 1 + bias_value = self.calculate_bias(i, length, bias_type) + max_value += bias_value + + if global_position in overlap_set: + my_value += bias_value + + return my_value / max_value if max_value > 0 else 0.0 + + def ts_recall(self, y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): + """Calculate Recall for time series anomaly detection.""" + if isinstance(y_real[0], tuple): + total_overlap_reward = 0.0 + total_cardinality = 0 + + for real_range in y_real: + overlap_set = set() + cardinality = 0 + + for pred_start, pred_end in y_pred: + overlap_start = max(real_range[0], pred_start) + overlap_end = min(real_range[1], pred_end) + + if overlap_start <= overlap_end: + overlap_set.update(range(overlap_start, overlap_end + 1)) + cardinality += 1 + + if overlap_set: + overlap_reward = self.calculate_overlap_reward_recall( + real_range, overlap_set, bias_type + ) + gamma_value = self.gamma_select(cardinality, gamma, udf_gamma) + + total_overlap_reward += gamma_value * overlap_reward + total_cardinality += 1 + + return total_overlap_reward / len(y_real) if y_real else 0.0 + + # Handle multiple sets of y_real + elif ( + isinstance(y_real, list) and len(y_real) > 0 and isinstance(y_real[0], list) + ): + total_recall = 0.0 + total_real = 0 + + for real_ranges in y_pred: # Iterate over all sets of real ranges + precision = self.ts_recall( + real_ranges, y_real, gamma, bias_type, udf_gamma + ) + total_recall += precision * len(real_ranges) + total_real += len(real_ranges) + + return total_recall / total_real if total_real > 0 else 0.0 From d4dc5cafc65e1145971ac9eb5c55fad2ae958fa6 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 3 Dec 2024 10:35:42 +0530 Subject: [PATCH 04/21] merged into into one file --- .../anomaly_detection/range_metrics.py | 261 ++++++++++++++++++ .../metrics/anomaly_detection/ts_precision.py | 170 ------------ .../metrics/anomaly_detection/ts_recall.py | 148 ---------- 3 files changed, 261 insertions(+), 318 deletions(-) create mode 100644 aeon/benchmarking/metrics/anomaly_detection/range_metrics.py delete mode 100644 aeon/benchmarking/metrics/anomaly_detection/ts_precision.py delete mode 100644 aeon/benchmarking/metrics/anomaly_detection/ts_recall.py diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py new file mode 100644 index 0000000000..e24c75e463 --- /dev/null +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -0,0 +1,261 @@ +"""Calculate Precision, Recall, and F1-Score for time series anomaly detection.""" + +__all__ = ["ts_precision", "ts_recall"] + + +def __init__(self, bias="flat", alpha=0.0, gamma=None): + assert gamma in ["reciprocal", "one", "udf_gamma"], "Invalid gamma type" + assert bias in ["flat", "front", "middle", "back"], "Invalid bias type" + + self.bias = bias + self.alpha = alpha + self.gamma = gamma + + +def calculate_bias(position, length, bias_type="flat"): + """Calculate bias value based on position and length. + + Parameters + ---------- + position : int + Current position in the range + length : int + Total length of the range + bias_type : str + Type of bias to apply, Should be one of ["flat", "front", "middle", "back"]. + (default: "flat") + """ + if bias_type == "flat": + return 1.0 + elif bias_type == "front": + return 1.0 - (position - 1) / length + elif bias_type == "middle": + return 1.0 - abs(2 * (position - 1) / (length - 1) - 1) if length > 1 else 1.0 + elif bias_type == "back": + return position / length + else: + raise ValueError(f"Invalid bias type: {bias_type}") + + +def gamma_select(cardinality, gamma, udf_gamma=None): + """Select a gamma value based on the cardinality type.""" + if gamma == "one": + return 1.0 + elif gamma == "reciprocal": + return 1 / cardinality if cardinality > 1 else 1.0 + elif gamma == "udf_gamma": + if udf_gamma is not None: + return 1.0 / udf_gamma + else: + raise ValueError("udf_gamma must be provided for 'udf_gamma' gamma type.") + else: + raise ValueError("Invalid gamma type.") + + +def calculate_overlap_reward_precision(pred_range, overlap_set, bias_type): + """Overlap Reward for y_pred. + + Parameters + ---------- + pred_range : tuple + The predicted range. + overlap_set : set + The set of overlapping positions. + bias_type : str + Type of bias to apply, Should be one of ["flat", "front", "middle", "back"]. + + Returns + ------- + float + The weighted value for overlapping positions only. + """ + start, end = pred_range + length = end - start + 1 + + max_value = 0 # Total possible weighted value for all positions. + my_value = 0 # Weighted value for overlapping positions only. + + for i in range(1, length + 1): + global_position = start + i - 1 + bias_value = calculate_bias(i, length, bias_type) + max_value += bias_value + + if global_position in overlap_set: + my_value += bias_value + + return my_value / max_value if max_value > 0 else 0.0 + + +def calculate_overlap_reward_recall(real_range, overlap_set, bias_type): + """Overlap Reward for y_real. + + Parameters + ---------- + real_range : tuple + The real range. + overlap_set : set + The set of overlapping positions. + bias_type : str + Type of bias to apply, Should be one of ["flat", "front", "middle", "back"]. + + Returns + ------- + float + The weighted value for overlapping positions only. + """ + start, end = real_range + length = end - start + 1 + + max_value = 0.0 # Total possible weighted value for all positions. + my_value = 0.0 # Weighted value for overlapping positions only. + + for i in range(1, length + 1): + global_position = start + i - 1 + bias_value = calculate_bias(i, length, bias_type) + max_value += bias_value + + if global_position in overlap_set: + my_value += bias_value + + return my_value / max_value if max_value > 0 else 0.0 + + +def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): + """Precision for either a single set or the entire time series. + + Parameters + ---------- + y_pred : list of tuples or list of list of tuples + The predicted ranges. + y_real : list of tuples + The real ranges. + gamma : str + Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. + (default: "one") + bias_type : str + Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. + (default: "flat") + udf_gamma : int or None + User-defined gamma value. (default: None) + + Returns + ------- + float + Range-based precision + """ + """ + example: + y_pred = [(1, 3), (5, 7)] + y_real = [(2, 6), (8, 10)] + """ + # Check if the input is a single set of predicted ranges or multiple sets + if isinstance(y_pred[0], tuple): + # y_pred is a single set of predicted ranges + total_overlap_reward = 0.0 + total_cardinality = 0 + + for pred_range in y_pred: + overlap_set = set() + cardinality = 0 + + for real_start, real_end in y_real: + overlap_start = max(pred_range[0], real_start) + overlap_end = min(pred_range[1], real_end) + + if overlap_start <= overlap_end: + overlap_set.update(range(overlap_start, overlap_end + 1)) + cardinality += 1 + + overlap_reward = calculate_overlap_reward_precision( + pred_range, overlap_set, bias_type + ) + gamma_value = gamma_select(cardinality, gamma, udf_gamma) + + total_overlap_reward += gamma_value * overlap_reward + total_cardinality += 1 + + return ( + total_overlap_reward / total_cardinality if total_cardinality > 0 else 0.0 + ) + + else: + """ + example: + y_pred = [[(1, 3), (5, 7)],[(10, 12)]] + y_real = [(2, 6), (8, 10)] + """ + # y_pred as multiple sets of predicted ranges + total_precision = 0.0 + total_ranges = 0 + + for pred_ranges in y_pred: # Iterate over all sets of predicted ranges + precision = ts_precision( + pred_ranges, y_real, gamma, bias_type, udf_gamma + ) # Recursive call for single sets + total_precision += precision * len(pred_ranges) + total_ranges += len(pred_ranges) + + return total_precision / total_ranges if total_ranges > 0 else 0.0 + + +def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): + """Calculate Recall for time series anomaly detection. + + Parameters + ---------- + y_pred : list of tuples or list of list of tuples + The predicted ranges. + y_real : list of tuples + The real ranges. + gamma : str + Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. + (default: "one") + bias_type : str + Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. + (default: "flat") + udf_gamma : int or None + User-defined gamma value. (default: None) + + Returns + ------- + float + Range-based recall + """ + if isinstance(y_real[0], tuple): + total_overlap_reward = 0.0 + total_cardinality = 0 + + for real_range in y_real: + overlap_set = set() + cardinality = 0 + + for pred_start, pred_end in y_pred: + overlap_start = max(real_range[0], pred_start) + overlap_end = min(real_range[1], pred_end) + + if overlap_start <= overlap_end: + overlap_set.update(range(overlap_start, overlap_end + 1)) + cardinality += 1 + + if overlap_set: + overlap_reward = calculate_overlap_reward_recall( + real_range, overlap_set, bias_type + ) + gamma_value = gamma_select(cardinality, gamma, udf_gamma) + + total_overlap_reward += gamma_value * overlap_reward + total_cardinality += 1 + + return total_overlap_reward / len(y_real) if y_real else 0.0 + + # Handle multiple sets of y_real + elif isinstance(y_real, list) and len(y_real) > 0 and isinstance(y_real[0], list): + total_recall = 0.0 + total_real = 0 + + for real_ranges in y_pred: # Iterate over all sets of real ranges + precision = ts_recall(real_ranges, y_real, gamma, bias_type, udf_gamma) + total_recall += precision * len(real_ranges) + total_real += len(real_ranges) + + return total_recall / total_real if total_real > 0 else 0.0 diff --git a/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py b/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py deleted file mode 100644 index 14cb519f7a..0000000000 --- a/aeon/benchmarking/metrics/anomaly_detection/ts_precision.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Calculate the precision of a time series anomaly detection model.""" - - -class RangePrecision: - """ - Calculates Precision for time series. - - Parameters - ---------- - y_real : np.ndarray - set of ground truth anomaly ranges (actual anomalies). - - y_pred : np.ndarray - set of predicted anomaly ranges. - - cardinality : str, default="one" - Number of overlaps between y_pred and y_real. - - gamma : float, default=1.0 - Overlpa Cardinality Factor. Penalizes or adjusts the metric based on - the cardinality. - Should be one of {'reciprocal', 'one', 'udf_gamma'}. - - alpha : float - Weight of the existence reward. Because precision by definition emphasizes on - prediction quality, there is no need for an existence reward and this value - should always be set to 0. - - bias : str, default="flat" - Captures importance of positional factors within anomaly ranges. - Determines the weight given to specific portions of anomaly range - when calculating overlap rewards. - Should be one of {'flat', 'front', 'middle', 'back'}. - - 'flat' - All positions are equally important. - 'front' - Front positions are more important. - 'middle' - Middle positions are more important. - 'back' - Back positions are more important. - - omega : float - Measure the extent and overlap between y_pred and y_real. - Considers the size and position of overlap and rewards. Should - be a float value between 0 and 1. - """ - - def __init__(self, bias="flat", alpha=0.0, gamma=None): - assert gamma in ["reciprocal", "one", "udf_gamma"], "Invalid gamma type" - assert bias in ["flat", "front", "middle", "back"], "Invalid bias type" - - self.bias = bias - self.alpha = alpha - self.gamma = gamma - - def calculate_bias(self, position, length, bias_type="flat"): - """Calculate bias value based on position and length. - - Args: - position: Current position in the range - length: Total length of the range - bias_type: Type of bias to apply (default: "flat") - """ - if bias_type == "flat": - return 1.0 - elif bias_type == "front": - return 1.0 - (position - 1) / length - elif bias_type == "middle": - return ( - 1.0 - abs(2 * (position - 1) / (length - 1) - 1) if length > 1 else 1.0 - ) - elif bias_type == "back": - return position / length - else: - return 1.0 - - def gamma_select(self, cardinality, gamma: str, udf_gamma=None) -> float: - """Select a gamma value based on the cardinality type.""" - if gamma == "one": - return 1.0 - elif gamma == "reciprocal": - if cardinality > 1: - return 1 / cardinality - else: - return 1.0 - elif gamma == "udf_gamma": - if udf_gamma is not None: - return 1.0 / udf_gamma - else: - raise ValueError( - "udf_gamma must be provided for 'udf_gamma' gamma type." - ) - else: - raise ValueError("Invalid gamma type") - - def calculate_overlap_reward_precision(self, pred_range, overlap_set, bias_type): - """Overlap Reward for y_pred.""" - start, end = pred_range - length = end - start + 1 - - max_value = 0 # Total possible weighted value for all positions. - my_value = 0 # Weighted value for overlapping positions only. - - for i in range(1, length + 1): - global_position = start + i - 1 - bias_value = self.calculate_bias(i, length, bias_type) - max_value += bias_value - - if global_position in overlap_set: - my_value += bias_value - - return my_value / max_value if max_value > 0 else 0.0 - - def ts_precision( - self, y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None - ): - """Precision for either a single set or the entire time series. - - example: - y_pred = [(1, 3), (5, 7)] - y_real = [(2, 6), (8, 10)] - """ - # Check if the input is a single set of predicted ranges or multiple sets - if isinstance(y_pred[0], tuple): - # y_pred is a single set of predicted ranges - total_overlap_reward = 0.0 - total_cardinality = 0 - - for pred_range in y_pred: - overlap_set = set() - cardinality = 0 - - for real_start, real_end in y_real: - overlap_start = max(pred_range[0], real_start) - overlap_end = min(pred_range[1], real_end) - - if overlap_start <= overlap_end: - overlap_set.update(range(overlap_start, overlap_end + 1)) - cardinality += 1 - - overlap_reward = self.calculate_overlap_reward_precision( - pred_range, overlap_set, bias_type - ) - gamma_value = self.gamma_select(cardinality, gamma, udf_gamma) - - total_overlap_reward += gamma_value * overlap_reward - total_cardinality += 1 - - return ( - total_overlap_reward / total_cardinality - if total_cardinality > 0 - else 0.0 - ) - - else: - """ - example: - y_pred = [[(1, 3), (5, 7)],[(10, 12)]] - y_real = [(2, 6), (8, 10)] - """ - # y_pred as multiple sets of predicted ranges - total_precision = 0.0 - total_ranges = 0 - - for pred_ranges in y_pred: # Iterate over all sets of predicted ranges - precision = self.ts_precision( - pred_ranges, y_real, gamma, bias_type, udf_gamma - ) # Recursive call for single sets - total_precision += precision * len(pred_ranges) - total_ranges += len(pred_ranges) - - return total_precision / total_ranges if total_ranges > 0 else 0.0 diff --git a/aeon/benchmarking/metrics/anomaly_detection/ts_recall.py b/aeon/benchmarking/metrics/anomaly_detection/ts_recall.py deleted file mode 100644 index 6d204c49f4..0000000000 --- a/aeon/benchmarking/metrics/anomaly_detection/ts_recall.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Calculate the Recall metric of a time series anomaly detection model.""" - - -class RangeRecall: - """Calculates Recall for time series. - - Parameters - ---------- - y_real : np.ndarray - Set of ground truth anomaly ranges (actual anomalies). - - y_pred : np.ndarray - Set of predicted anomaly ranges. - - cardinality : str, default="one" - Number of overlaps between y_pred and y_real. - - gamma : float, default=1.0 - Overlap Cardinality Factor. Penalizes or adjusts the metric based on - the cardinality. - Should be one of {'reciprocal', 'one', 'udf_gamma'}. - - alpha : float - Weight of the existence reward. Since Recall emphasizes coverage, - you might adjust this value if needed. - - bias : str, default="flat" - Captures the importance of positional factors within anomaly ranges. - Determines the weight given to specific portions of anomaly range - when calculating overlap rewards. - Should be one of {'flat', 'front', 'middle', 'back'}. - - omega : float - Measure the extent and overlap between y_pred and y_real. - Considers the size and position of overlap and rewards. Should - be a float value between 0 and 1. - """ - - def _init_(self, bias="flat", alpha=0.0, gamma="one"): - assert gamma in ["reciprocal", "one", "udf_gamma"], "Invalid gamma type" - assert bias in ["flat", "front", "middle", "back"], "Invalid bias type" - - self.bias = bias - self.alpha = alpha - self.gamma = gamma - - def calculate_bias(self, position, length, bias_type="flat"): - """Calculate bias value based on position and length. - - Args: - position: Current position in the range - length: Total length of the range - bias_type: Type of bias to apply (default: "flat") - """ - if bias_type == "flat": - return 1.0 - elif bias_type == "front": - return 1.0 - (position - 1) / length - elif bias_type == "middle": - return ( - 1.0 - abs(2 * (position - 1) / (length - 1) - 1) if length > 1 else 1.0 - ) - elif bias_type == "back": - return position / length - else: - return 1.0 - - def gamma_select(self, cardinality, gamma: str, udf_gamma=None) -> float: - """Select a gamma value based on the cardinality type.""" - if gamma == "one": - return 1.0 - elif gamma == "reciprocal": - if cardinality > 1: - return 1 / cardinality - else: - return 1.0 - elif gamma == "udf_gamma": - if udf_gamma is not None: - return 1.0 / udf_gamma - else: - raise ValueError( - "udf_gamma must be provided for 'udf_gamma' gamma type." - ) - else: - raise ValueError("Invalid gamma type") - - def calculate_overlap_reward_recall(self, real_range, overlap_set, bias_type): - """Overlap Reward for y_real.""" - start, end = real_range - length = end - start + 1 - - max_value = 0.0 # Total possible weighted value for all positions. - my_value = 0.0 # Weighted value for overlapping positions only. - - for i in range(1, length + 1): - global_position = start + i - 1 - bias_value = self.calculate_bias(i, length, bias_type) - max_value += bias_value - - if global_position in overlap_set: - my_value += bias_value - - return my_value / max_value if max_value > 0 else 0.0 - - def ts_recall(self, y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): - """Calculate Recall for time series anomaly detection.""" - if isinstance(y_real[0], tuple): - total_overlap_reward = 0.0 - total_cardinality = 0 - - for real_range in y_real: - overlap_set = set() - cardinality = 0 - - for pred_start, pred_end in y_pred: - overlap_start = max(real_range[0], pred_start) - overlap_end = min(real_range[1], pred_end) - - if overlap_start <= overlap_end: - overlap_set.update(range(overlap_start, overlap_end + 1)) - cardinality += 1 - - if overlap_set: - overlap_reward = self.calculate_overlap_reward_recall( - real_range, overlap_set, bias_type - ) - gamma_value = self.gamma_select(cardinality, gamma, udf_gamma) - - total_overlap_reward += gamma_value * overlap_reward - total_cardinality += 1 - - return total_overlap_reward / len(y_real) if y_real else 0.0 - - # Handle multiple sets of y_real - elif ( - isinstance(y_real, list) and len(y_real) > 0 and isinstance(y_real[0], list) - ): - total_recall = 0.0 - total_real = 0 - - for real_ranges in y_pred: # Iterate over all sets of real ranges - precision = self.ts_recall( - real_ranges, y_real, gamma, bias_type, udf_gamma - ) - total_recall += precision * len(real_ranges) - total_real += len(real_ranges) - - return total_recall / total_real if total_real > 0 else 0.0 From 4db80272a71c3919ee297522a85e85e48ce5f272 Mon Sep 17 00:00:00 2001 From: Aryan Date: Fri, 20 Dec 2024 12:30:32 +0530 Subject: [PATCH 05/21] test added --- .../metrics/anomaly_detection/__init__.py | 8 +++ .../anomaly_detection/range_metrics.py | 50 ++++++++++---- .../anomaly_detection/tests/test_metrics.py | 69 +++++++++++++++++++ 3 files changed, 112 insertions(+), 15 deletions(-) create mode 100644 aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py diff --git a/aeon/benchmarking/metrics/anomaly_detection/__init__.py b/aeon/benchmarking/metrics/anomaly_detection/__init__.py index fdbf13cec9..cf6ccac42c 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/__init__.py +++ b/aeon/benchmarking/metrics/anomaly_detection/__init__.py @@ -14,6 +14,9 @@ "range_pr_auc_score", "range_pr_vus_score", "range_roc_vus_score", + "ts_precision", + "ts_recall", + "ts_fscore", ] from aeon.benchmarking.metrics.anomaly_detection._binary import ( @@ -35,3 +38,8 @@ range_roc_auc_score, range_roc_vus_score, ) +from aeon.benchmarking.metrics.anomaly_detection.range_metrics import ( + ts_fscore, + ts_precision, + ts_recall, +) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index e24c75e463..c8b7fd5934 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -1,6 +1,6 @@ """Calculate Precision, Recall, and F1-Score for time series anomaly detection.""" -__all__ = ["ts_precision", "ts_recall"] +__all__ = ["ts_precision", "ts_recall", "ts_fscore"] def __init__(self, bias="flat", alpha=0.0, gamma=None): @@ -198,14 +198,14 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): return total_precision / total_ranges if total_ranges > 0 else 0.0 -def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): +def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None): """Calculate Recall for time series anomaly detection. Parameters ---------- y_pred : list of tuples or list of list of tuples The predicted ranges. - y_real : list of tuples + y_real : list of tuples or list of list of tuples The real ranges. gamma : str Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. @@ -213,6 +213,8 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): bias_type : str Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. (default: "flat") + alpha : float + Weight for existence reward in recall calculation. (default: 0.0) udf_gamma : int or None User-defined gamma value. (default: None) @@ -221,41 +223,59 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): float Range-based recall """ - if isinstance(y_real[0], tuple): + if isinstance(y_real[0], tuple): # Single set of real ranges total_overlap_reward = 0.0 - total_cardinality = 0 for real_range in y_real: overlap_set = set() cardinality = 0 - for pred_start, pred_end in y_pred: - overlap_start = max(real_range[0], pred_start) - overlap_end = min(real_range[1], pred_end) + for pred_range in y_pred: + overlap_start = max(real_range[0], pred_range[0]) + overlap_end = min(real_range[1], pred_range[1]) if overlap_start <= overlap_end: overlap_set.update(range(overlap_start, overlap_end + 1)) cardinality += 1 + # Existence Reward + existence_reward = 1.0 if overlap_set else 0.0 + if overlap_set: overlap_reward = calculate_overlap_reward_recall( real_range, overlap_set, bias_type ) gamma_value = gamma_select(cardinality, gamma, udf_gamma) + overlap_reward *= gamma_value + else: + overlap_reward = 0.0 - total_overlap_reward += gamma_value * overlap_reward - total_cardinality += 1 + # Total Recall Score + recall_score = alpha * existence_reward + (1 - alpha) * overlap_reward + total_overlap_reward += recall_score return total_overlap_reward / len(y_real) if y_real else 0.0 - # Handle multiple sets of y_real - elif isinstance(y_real, list) and len(y_real) > 0 and isinstance(y_real[0], list): + elif isinstance(y_real[0], list): # Multiple sets of real ranges total_recall = 0.0 total_real = 0 - for real_ranges in y_pred: # Iterate over all sets of real ranges - precision = ts_recall(real_ranges, y_real, gamma, bias_type, udf_gamma) - total_recall += precision * len(real_ranges) + for real_ranges in y_real: # Iterate over all sets of real ranges + recall = ts_recall(y_pred, real_ranges, gamma, bias_type, alpha, udf_gamma) + total_recall += recall * len(real_ranges) total_real += len(real_ranges) return total_recall / total_real if total_real > 0 else 0.0 + + +def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None): + """Calculate F1-Score for time series anomaly detection.""" + precision = ts_precision(y_pred, y_real, gamma, bias_type, udf_gamma) + recall = ts_recall(y_pred, y_real, gamma, bias_type, alpha, udf_gamma) + + if precision + recall > 0: + fscore = 2 * (precision * recall) / (precision + recall) + else: + fscore = 0.0 + + return fscore diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py new file mode 100644 index 0000000000..0c202a283d --- /dev/null +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -0,0 +1,69 @@ +"""Test cases for metrics.""" + +import numpy as np + +from aeon.benchmarking.metrics.anomaly_detection.range_metrics import ( + ts_fscore, + ts_precision, + ts_recall, +) + +# Single Overlapping Range +y_pred = [(1, 4)] +y_real = [(2, 6)] + +precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") +recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) +f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) + +np.testing.assert_almost_equal(precision, 0.750000, decimal=6) +np.testing.assert_almost_equal(recall, 0.600000, decimal=6) +np.testing.assert_almost_equal(f1_score, 0.666667, decimal=6) + +# Multiple Non-Overlapping Ranges +y_pred = [(1, 2), (7, 8)] +y_real = [(3, 4), (9, 10)] + +precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") +recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) +f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) + +np.testing.assert_almost_equal(precision, 0.000000, decimal=6) +np.testing.assert_almost_equal(recall, 0.000000, decimal=6) +np.testing.assert_almost_equal(f1_score, 0.000000, decimal=6) + +# Multiple Overlapping Ranges +y_pred = [(1, 3), (5, 7)] +y_real = [(2, 6), (8, 10)] + +precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") +recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) +f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) + +np.testing.assert_almost_equal(precision, 0.666667, decimal=6) +np.testing.assert_almost_equal(recall, 0.5, decimal=6) +np.testing.assert_almost_equal(f1_score, 0.571429, decimal=6) + +# Nested Lists of Predictions +y_pred = [[(1, 3), (5, 7)], [(10, 12)]] +y_real = [(2, 6), (8, 10)] + +precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") +recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) +f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) + +np.testing.assert_almost_equal(precision, 0.555556, decimal=6) +np.testing.assert_almost_equal(recall, 0.555556, decimal=6) +np.testing.assert_almost_equal(f1_score, 0.555556, decimal=6) + +# All Encompassing Range +y_pred = [(1, 10)] +y_real = [(2, 3), (5, 6), (8, 9)] + +precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") +recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) +f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) + +np.testing.assert_almost_equal(precision, 0.600000, decimal=6) +np.testing.assert_almost_equal(recall, 1.000000, decimal=6) +np.testing.assert_almost_equal(f1_score, 0.75, decimal=6) From c09873192f7524349d9ecdb8b2d44a9b0a99875a Mon Sep 17 00:00:00 2001 From: Aryan Date: Sun, 29 Dec 2024 08:01:34 +0530 Subject: [PATCH 06/21] Changes in test and range_metrics --- .../anomaly_detection/range_metrics.py | 10 +- .../anomaly_detection/tests/test_metrics.py | 128 ++++++++++-------- 2 files changed, 70 insertions(+), 68 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index c8b7fd5934..75216e65e4 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -1,17 +1,9 @@ """Calculate Precision, Recall, and F1-Score for time series anomaly detection.""" +maintainer = [] __all__ = ["ts_precision", "ts_recall", "ts_fscore"] -def __init__(self, bias="flat", alpha=0.0, gamma=None): - assert gamma in ["reciprocal", "one", "udf_gamma"], "Invalid gamma type" - assert bias in ["flat", "front", "middle", "back"], "Invalid bias type" - - self.bias = bias - self.alpha = alpha - self.gamma = gamma - - def calculate_bias(position, length, bias_type="flat"): """Calculate bias value based on position and length. diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index 0c202a283d..a695787af3 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -1,6 +1,7 @@ -"""Test cases for metrics.""" +"""Test cases for the range-based anomaly detection metrics.""" import numpy as np +import pytest from aeon.benchmarking.metrics.anomaly_detection.range_metrics import ( ts_fscore, @@ -8,62 +9,71 @@ ts_recall, ) -# Single Overlapping Range -y_pred = [(1, 4)] -y_real = [(2, 6)] -precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") -recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) -f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) - -np.testing.assert_almost_equal(precision, 0.750000, decimal=6) -np.testing.assert_almost_equal(recall, 0.600000, decimal=6) -np.testing.assert_almost_equal(f1_score, 0.666667, decimal=6) - -# Multiple Non-Overlapping Ranges -y_pred = [(1, 2), (7, 8)] -y_real = [(3, 4), (9, 10)] - -precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") -recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) -f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) - -np.testing.assert_almost_equal(precision, 0.000000, decimal=6) -np.testing.assert_almost_equal(recall, 0.000000, decimal=6) -np.testing.assert_almost_equal(f1_score, 0.000000, decimal=6) - -# Multiple Overlapping Ranges -y_pred = [(1, 3), (5, 7)] -y_real = [(2, 6), (8, 10)] - -precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") -recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) -f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) - -np.testing.assert_almost_equal(precision, 0.666667, decimal=6) -np.testing.assert_almost_equal(recall, 0.5, decimal=6) -np.testing.assert_almost_equal(f1_score, 0.571429, decimal=6) - -# Nested Lists of Predictions -y_pred = [[(1, 3), (5, 7)], [(10, 12)]] -y_real = [(2, 6), (8, 10)] - -precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") -recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) -f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) - -np.testing.assert_almost_equal(precision, 0.555556, decimal=6) -np.testing.assert_almost_equal(recall, 0.555556, decimal=6) -np.testing.assert_almost_equal(f1_score, 0.555556, decimal=6) - -# All Encompassing Range -y_pred = [(1, 10)] -y_real = [(2, 3), (5, 6), (8, 9)] - -precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") -recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) -f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) - -np.testing.assert_almost_equal(precision, 0.600000, decimal=6) -np.testing.assert_almost_equal(recall, 1.000000, decimal=6) -np.testing.assert_almost_equal(f1_score, 0.75, decimal=6) +# Test cases for metrics +@pytest.mark.parametrize( + "y_pred, y_real, expected_precision, expected_recall, expected_f1", + [ + ([(1, 4)], [(2, 6)], 0.750000, 0.600000, 0.666667), # Single Overlapping Range + ( + [(1, 2), (7, 8)], + [(3, 4), (9, 10)], + 0.000000, + 0.000000, + 0.000000, + ), # Multiple Non-Overlapping Ranges + ( + [(1, 3), (5, 7)], + [(2, 6), (8, 10)], + 0.666667, + 0.500000, + 0.571429, + ), # Multiple Overlapping Ranges + ( + [[(1, 3), (5, 7)], [(10, 12)]], + [(2, 6), (8, 10)], + 0.555556, + 0.555556, + 0.555556, + ), # Nested Lists of Predictions + ( + [(1, 10)], + [(2, 3), (5, 6), (8, 9)], + 0.600000, + 1.000000, + 0.750000, + ), # All Encompassing Range + ( + [(1, 2)], + [(1, 1)], + 0.5, + 1.000000, + 0.666667, + ), # Converted Binary to Range-Based(Existing example) + ], +) +def test_metrics(y_pred, y_real, expected_precision, expected_recall, expected_f1): + """Test the range-based anomaly detection metrics.""" + precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") + recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) + f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) + + # Use assertions with detailed error messages for debugging + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=f"Precision failed! Expected={expected_precision}, Got={precision}", + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=f"Recall failed! Expected={expected_recall}, Got={recall}", + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=f"F1-Score failed! Expected={expected_f1}, Got={f1_score}", + ) From 497362f770e8a88fae1a5cefef2b7d7abe05ac9e Mon Sep 17 00:00:00 2001 From: Aryan Date: Sun, 29 Dec 2024 10:35:25 +0530 Subject: [PATCH 07/21] list of list running but error! --- .../anomaly_detection/range_metrics.py | 22 ++++++++++++++----- .../anomaly_detection/tests/test_metrics.py | 6 ++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 75216e65e4..6b26ae9e5b 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -223,12 +223,22 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm cardinality = 0 for pred_range in y_pred: - overlap_start = max(real_range[0], pred_range[0]) - overlap_end = min(real_range[1], pred_range[1]) - - if overlap_start <= overlap_end: - overlap_set.update(range(overlap_start, overlap_end + 1)) - cardinality += 1 + # Handle nested lists in y_pred + if isinstance(pred_range, list): + for sub_pred_range in pred_range: + overlap_start = max(real_range[0], sub_pred_range[0]) + overlap_end = min(real_range[1], sub_pred_range[1]) + + if overlap_start <= overlap_end: + overlap_set.update(range(overlap_start, overlap_end + 1)) + cardinality += 1 + else: # Handle flat list of tuples + overlap_start = max(real_range[0], pred_range[0]) + overlap_end = min(real_range[1], pred_range[1]) + + if overlap_start <= overlap_end: + overlap_set.update(range(overlap_start, overlap_end + 1)) + cardinality += 1 # Existence Reward existence_reward = 1.0 if overlap_set else 0.0 diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index a695787af3..c1fc47940b 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -25,16 +25,16 @@ ( [(1, 3), (5, 7)], [(2, 6), (8, 10)], + 0.5, 0.666667, - 0.500000, 0.571429, ), # Multiple Overlapping Ranges ( [[(1, 3), (5, 7)], [(10, 12)]], [(2, 6), (8, 10)], + 0.625, 0.555556, - 0.555556, - 0.555556, + 0.588235, ), # Nested Lists of Predictions ( [(1, 10)], From ab87680654403ffb2cc6df9d67cdb51deb34c10c Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 31 Dec 2024 00:06:07 +0530 Subject: [PATCH 08/21] flattening lists, all cases passed --- .../anomaly_detection/range_metrics.py | 281 +++++++----------- .../anomaly_detection/tests/test_metrics.py | 4 +- 2 files changed, 112 insertions(+), 173 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 6b26ae9e5b..1561581160 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -4,6 +4,31 @@ __all__ = ["ts_precision", "ts_recall", "ts_fscore"] +def flatten_ranges(ranges): + """ + If the input is a list of lists, it flattens it into a single list. + + Parameters + ---------- + ranges : list of tuples or list of lists of tuples + The ranges to flatten. + + Returns + ------- + list of tuples + A flattened list of ranges. + """ + if not ranges: + return [] + if isinstance(ranges[0], list): + flat = [] + for sublist in ranges: + for pred in sublist: + flat.append(pred) + return flat + return ranges + + def calculate_bias(position, length, bias_type="flat"): """Calculate bias value based on position and length. @@ -44,236 +69,150 @@ def gamma_select(cardinality, gamma, udf_gamma=None): raise ValueError("Invalid gamma type.") -def calculate_overlap_reward_precision(pred_range, overlap_set, bias_type): - """Overlap Reward for y_pred. +def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): + """ + Calculate Global Precision for time series anomaly detection. Parameters ---------- - pred_range : tuple - The predicted range. - overlap_set : set - The set of overlapping positions. + y_pred : list of tuples or list of lists of tuples + The predicted ranges. + y_real : list of tuples or list of lists of tuples + The real (actual) ranges. + gamma : str + Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. + (default: "one") bias_type : str - Type of bias to apply, Should be one of ["flat", "front", "middle", "back"]. + Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. + (default: "flat") + udf_gamma : int or None + User-defined gamma value. (default: None) Returns ------- float - The weighted value for overlapping positions only. + Global Precision """ - start, end = pred_range - length = end - start + 1 - - max_value = 0 # Total possible weighted value for all positions. - my_value = 0 # Weighted value for overlapping positions only. - - for i in range(1, length + 1): - global_position = start + i - 1 - bias_value = calculate_bias(i, length, bias_type) - max_value += bias_value - - if global_position in overlap_set: - my_value += bias_value - - return my_value / max_value if max_value > 0 else 0.0 + # Flattening y_pred and y_real to resolve nested lists + flat_y_pred = flatten_ranges(y_pred) + flat_y_real = flatten_ranges(y_real) + overlapping_weighted_positions = 0.0 + total_pred_weight = 0.0 -def calculate_overlap_reward_recall(real_range, overlap_set, bias_type): - """Overlap Reward for y_real. + for pred_range in flat_y_pred: + start_pred, end_pred = pred_range + length_pred = end_pred - start_pred + 1 - Parameters - ---------- - real_range : tuple - The real range. - overlap_set : set - The set of overlapping positions. - bias_type : str - Type of bias to apply, Should be one of ["flat", "front", "middle", "back"]. - - Returns - ------- - float - The weighted value for overlapping positions only. - """ - start, end = real_range - length = end - start + 1 + for i in range(1, length_pred + 1): + pos = start_pred + i - 1 + bias = calculate_bias(i, length_pred, bias_type) - max_value = 0.0 # Total possible weighted value for all positions. - my_value = 0.0 # Weighted value for overlapping positions only. + # Check if the position is in any real range + in_real = any( + real_start <= pos <= real_end for real_start, real_end in flat_y_real + ) - for i in range(1, length + 1): - global_position = start + i - 1 - bias_value = calculate_bias(i, length, bias_type) - max_value += bias_value + if in_real: + gamma_value = gamma_select(1, gamma, udf_gamma) + overlapping_weighted_positions += bias * gamma_value - if global_position in overlap_set: - my_value += bias_value + total_pred_weight += bias - return my_value / max_value if max_value > 0 else 0.0 + precision = ( + overlapping_weighted_positions / total_pred_weight + if total_pred_weight > 0 + else 0.0 + ) + return precision -def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): - """Precision for either a single set or the entire time series. +def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None): + """ + Calculate Global Recall for time series anomaly detection. Parameters ---------- - y_pred : list of tuples or list of list of tuples + y_pred : list of tuples or list of lists of tuples The predicted ranges. - y_real : list of tuples - The real ranges. + y_real : list of tuples or list of lists of tuples + The real (actual) ranges. gamma : str Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. (default: "one") bias_type : str Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. (default: "flat") + alpha : float + Weight for existence reward in recall calculation. (default: 0.0) udf_gamma : int or None User-defined gamma value. (default: None) Returns ------- float - Range-based precision - """ - """ - example: - y_pred = [(1, 3), (5, 7)] - y_real = [(2, 6), (8, 10)] + Global Recall """ - # Check if the input is a single set of predicted ranges or multiple sets - if isinstance(y_pred[0], tuple): - # y_pred is a single set of predicted ranges - total_overlap_reward = 0.0 - total_cardinality = 0 - - for pred_range in y_pred: - overlap_set = set() - cardinality = 0 - - for real_start, real_end in y_real: - overlap_start = max(pred_range[0], real_start) - overlap_end = min(pred_range[1], real_end) - - if overlap_start <= overlap_end: - overlap_set.update(range(overlap_start, overlap_end + 1)) - cardinality += 1 - - overlap_reward = calculate_overlap_reward_precision( - pred_range, overlap_set, bias_type - ) - gamma_value = gamma_select(cardinality, gamma, udf_gamma) + # Flattening y_pred and y_real + flat_y_pred = flatten_ranges(y_pred) + flat_y_real = flatten_ranges(y_real) - total_overlap_reward += gamma_value * overlap_reward - total_cardinality += 1 + overlapping_weighted_positions = 0.0 + total_real_weight = 0.0 - return ( - total_overlap_reward / total_cardinality if total_cardinality > 0 else 0.0 - ) + for real_range in flat_y_real: + start_real, end_real = real_range + length_real = end_real - start_real + 1 - else: - """ - example: - y_pred = [[(1, 3), (5, 7)],[(10, 12)]] - y_real = [(2, 6), (8, 10)] - """ - # y_pred as multiple sets of predicted ranges - total_precision = 0.0 - total_ranges = 0 + for i in range(1, length_real + 1): + pos = start_real + i - 1 + bias = calculate_bias(i, length_real, bias_type) - for pred_ranges in y_pred: # Iterate over all sets of predicted ranges - precision = ts_precision( - pred_ranges, y_real, gamma, bias_type, udf_gamma - ) # Recursive call for single sets - total_precision += precision * len(pred_ranges) - total_ranges += len(pred_ranges) + # Check if the position is in any predicted range + in_pred = any( + pred_start <= pos <= pred_end for pred_start, pred_end in flat_y_pred + ) - return total_precision / total_ranges if total_ranges > 0 else 0.0 + if in_pred: + gamma_value = gamma_select(1, gamma, udf_gamma) + overlapping_weighted_positions += bias * gamma_value + total_real_weight += bias -def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None): - """Calculate Recall for time series anomaly detection. + recall = ( + overlapping_weighted_positions / total_real_weight + if total_real_weight > 0 + else 0.0 + ) + return recall + + +def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None): + """ + Calculate F1-Score for time series anomaly detection. Parameters ---------- - y_pred : list of tuples or list of list of tuples + y_pred : list of tuples or list of lists of tuples The predicted ranges. - y_real : list of tuples or list of list of tuples - The real ranges. + y_real : list of tuples or list of lists of tuples + The real (actual) ranges. gamma : str Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. (default: "one") bias_type : str Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. (default: "flat") - alpha : float - Weight for existence reward in recall calculation. (default: 0.0) udf_gamma : int or None User-defined gamma value. (default: None) Returns ------- float - Range-based recall + F1-Score """ - if isinstance(y_real[0], tuple): # Single set of real ranges - total_overlap_reward = 0.0 - - for real_range in y_real: - overlap_set = set() - cardinality = 0 - - for pred_range in y_pred: - # Handle nested lists in y_pred - if isinstance(pred_range, list): - for sub_pred_range in pred_range: - overlap_start = max(real_range[0], sub_pred_range[0]) - overlap_end = min(real_range[1], sub_pred_range[1]) - - if overlap_start <= overlap_end: - overlap_set.update(range(overlap_start, overlap_end + 1)) - cardinality += 1 - else: # Handle flat list of tuples - overlap_start = max(real_range[0], pred_range[0]) - overlap_end = min(real_range[1], pred_range[1]) - - if overlap_start <= overlap_end: - overlap_set.update(range(overlap_start, overlap_end + 1)) - cardinality += 1 - - # Existence Reward - existence_reward = 1.0 if overlap_set else 0.0 - - if overlap_set: - overlap_reward = calculate_overlap_reward_recall( - real_range, overlap_set, bias_type - ) - gamma_value = gamma_select(cardinality, gamma, udf_gamma) - overlap_reward *= gamma_value - else: - overlap_reward = 0.0 - - # Total Recall Score - recall_score = alpha * existence_reward + (1 - alpha) * overlap_reward - total_overlap_reward += recall_score - - return total_overlap_reward / len(y_real) if y_real else 0.0 - - elif isinstance(y_real[0], list): # Multiple sets of real ranges - total_recall = 0.0 - total_real = 0 - - for real_ranges in y_real: # Iterate over all sets of real ranges - recall = ts_recall(y_pred, real_ranges, gamma, bias_type, alpha, udf_gamma) - total_recall += recall * len(real_ranges) - total_real += len(real_ranges) - - return total_recall / total_real if total_real > 0 else 0.0 - - -def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None): - """Calculate F1-Score for time series anomaly detection.""" - precision = ts_precision(y_pred, y_real, gamma, bias_type, udf_gamma) - recall = ts_recall(y_pred, y_real, gamma, bias_type, alpha, udf_gamma) + precision = ts_precision(y_pred, y_real, gamma, bias_type, udf_gamma=udf_gamma) + recall = ts_recall(y_pred, y_real, gamma, bias_type, alpha, udf_gamma=udf_gamma) if precision + recall > 0: fscore = 2 * (precision * recall) / (precision + recall) diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index c1fc47940b..d30c1e873b 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -25,15 +25,15 @@ ( [(1, 3), (5, 7)], [(2, 6), (8, 10)], - 0.5, 0.666667, + 0.500000, 0.571429, ), # Multiple Overlapping Ranges ( [[(1, 3), (5, 7)], [(10, 12)]], [(2, 6), (8, 10)], - 0.625, 0.555556, + 0.625000, 0.588235, ), # Nested Lists of Predictions ( From c18af4fa3388850d50bbf9cdb3616988344433b4 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 31 Dec 2024 00:31:12 +0530 Subject: [PATCH 09/21] Empty-Commit From 9c235828600bbc58976ec9ed098b9fb303394bc7 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 14 Jan 2025 15:19:50 +0530 Subject: [PATCH 10/21] changes --- aeon/benchmarking/metrics/anomaly_detection/range_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 1561581160..6b26daa851 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -1,6 +1,6 @@ """Calculate Precision, Recall, and F1-Score for time series anomaly detection.""" -maintainer = [] +__maintainer__ = [] __all__ = ["ts_precision", "ts_recall", "ts_fscore"] From df42934c3959588cf494f5f50ca367a33997a304 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 14 Jan 2025 15:24:00 +0530 Subject: [PATCH 11/21] Protected functions --- .../anomaly_detection/range_metrics.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 6b26daa851..3dff50109b 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -4,7 +4,7 @@ __all__ = ["ts_precision", "ts_recall", "ts_fscore"] -def flatten_ranges(ranges): +def _flatten_ranges(ranges): """ If the input is a list of lists, it flattens it into a single list. @@ -29,7 +29,7 @@ def flatten_ranges(ranges): return ranges -def calculate_bias(position, length, bias_type="flat"): +def _calculate_bias(position, length, bias_type="flat"): """Calculate bias value based on position and length. Parameters @@ -54,7 +54,7 @@ def calculate_bias(position, length, bias_type="flat"): raise ValueError(f"Invalid bias type: {bias_type}") -def gamma_select(cardinality, gamma, udf_gamma=None): +def _gamma_select(cardinality, gamma, udf_gamma=None): """Select a gamma value based on the cardinality type.""" if gamma == "one": return 1.0 @@ -94,8 +94,8 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): Global Precision """ # Flattening y_pred and y_real to resolve nested lists - flat_y_pred = flatten_ranges(y_pred) - flat_y_real = flatten_ranges(y_real) + flat_y_pred = _flatten_ranges(y_pred) + flat_y_real = _flatten_ranges(y_real) overlapping_weighted_positions = 0.0 total_pred_weight = 0.0 @@ -106,7 +106,7 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): for i in range(1, length_pred + 1): pos = start_pred + i - 1 - bias = calculate_bias(i, length_pred, bias_type) + bias = _calculate_bias(i, length_pred, bias_type) # Check if the position is in any real range in_real = any( @@ -114,7 +114,7 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): ) if in_real: - gamma_value = gamma_select(1, gamma, udf_gamma) + gamma_value = _gamma_select(1, gamma, udf_gamma) overlapping_weighted_positions += bias * gamma_value total_pred_weight += bias @@ -154,8 +154,8 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm Global Recall """ # Flattening y_pred and y_real - flat_y_pred = flatten_ranges(y_pred) - flat_y_real = flatten_ranges(y_real) + flat_y_pred = _flatten_ranges(y_pred) + flat_y_real = _flatten_ranges(y_real) overlapping_weighted_positions = 0.0 total_real_weight = 0.0 @@ -166,7 +166,7 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm for i in range(1, length_real + 1): pos = start_real + i - 1 - bias = calculate_bias(i, length_real, bias_type) + bias = _calculate_bias(i, length_real, bias_type) # Check if the position is in any predicted range in_pred = any( @@ -174,7 +174,7 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm ) if in_pred: - gamma_value = gamma_select(1, gamma, udf_gamma) + gamma_value = _gamma_select(1, gamma, udf_gamma) overlapping_weighted_positions += bias * gamma_value total_real_weight += bias From b5bfab440a1ad662ebde928b7ea682d40ca4fc0b Mon Sep 17 00:00:00 2001 From: Aryan Date: Wed, 15 Jan 2025 15:05:42 +0530 Subject: [PATCH 12/21] Changes in documentation --- .../anomaly_detection/range_metrics.py | 115 +++++++++++++++--- .../anomaly_detection/tests/test_metrics.py | 2 +- 2 files changed, 99 insertions(+), 18 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 3dff50109b..a17e849bd6 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -11,12 +11,17 @@ def _flatten_ranges(ranges): Parameters ---------- ranges : list of tuples or list of lists of tuples - The ranges to flatten. + The ranges to flatten. each tuple shoulod be in the format of (start, end). Returns ------- list of tuples A flattened list of ranges. + + Examples + -------- + >>> _flatten_ranges([[(1, 5), (10, 15)], [(20, 25)]]) + [(1, 5), (10, 15), (20, 25)] """ if not ranges: return [] @@ -55,7 +60,29 @@ def _calculate_bias(position, length, bias_type="flat"): def _gamma_select(cardinality, gamma, udf_gamma=None): - """Select a gamma value based on the cardinality type.""" + """Select a gamma value based on the cardinality type. + + Parameters + ---------- + cardinality : int + The number of overlapping ranges. + gamma : str + Gamma to use. Should be one of ["one", "reciprocal", "udf_gamma"]. + udf_gamma : float or None, optional + The user-defined gamma value to use when `gamma` is set to "udf_gamma". + Required if `gamma` is "udf_gamma". + + Returns + ------- + float + The selected gamma value. + + Raises + ------ + ValueError + If an invalid `gamma` type is provided or if `udf_gamma` is required + but not provided. + """ if gamma == "one": return 1.0 elif gamma == "reciprocal": @@ -66,32 +93,48 @@ def _gamma_select(cardinality, gamma, udf_gamma=None): else: raise ValueError("udf_gamma must be provided for 'udf_gamma' gamma type.") else: - raise ValueError("Invalid gamma type.") + raise ValueError( + "Invalid gamma type. Choose from ['one', 'reciprocal', 'udf_gamma']." + ) -def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): +def ts_precision(y_pred, y_real, bias_type="flat"): """ Calculate Global Precision for time series anomaly detection. + Global Precision measures the proportion of correctly predicted anomaly positions + out of all all the predicted anomaly positions, aggregated across the entire time + series. + Parameters ---------- y_pred : list of tuples or list of lists of tuples - The predicted ranges. + The predicted anomaly ranges. + - Each tuple represents a range (start, end) of the anomaly where + start is starting index (inclusive) and end is ending index (inclusive). + - If y_pred is in the format of list of lists, they will be flattened into a \ + single list of tuples bringing it to the above format. y_real : list of tuples or list of lists of tuples - The real (actual) ranges. - gamma : str - Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. - (default: "one") + The real/actual (ground truth) ranges. + - Each tuple represents a range (start, end) of the anomaly where + start is starting index (inclusive) and end is ending index (inclusive). + - If y_real is in the format of list of lists, they will be flattened into a + single list of tuples bringing it to the above format. bias_type : str Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. (default: "flat") - udf_gamma : int or None - User-defined gamma value. (default: None) Returns ------- float Global Precision + + References + ---------- + .. [1] Tatbul, Nesime, Tae Jun Lee, Stan Zdonik, Mejbah Alam,and Justin Gottschlich. + "Precision and Recall for Time Series." 32nd Conference on Neural Information + Processing Systems (NeurIPS 2018), Montréal, Canada. + http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ # Flattening y_pred and y_real to resolve nested lists flat_y_pred = _flatten_ranges(y_pred) @@ -114,7 +157,8 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): ) if in_real: - gamma_value = _gamma_select(1, gamma, udf_gamma) + # For precision, gamma is fixed to "one" + gamma_value = 1.0 overlapping_weighted_positions += bias * gamma_value total_pred_weight += bias @@ -131,12 +175,24 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm """ Calculate Global Recall for time series anomaly detection. + Global Recall measures the proportion of correctly predicted anomaly positions + out of all the real/actual (ground truth) anomaly positions, aggregated across the + entire time series. + Parameters ---------- y_pred : list of tuples or list of lists of tuples - The predicted ranges. + The predicted anomaly ranges. + - Each tuple represents a range (start, end) of the anomaly where + start is starting index (inclusive) and end is ending index (inclusive). + - If y_pred is in the format of list of lists, they will be flattened into a + single list of tuples bringing it to the above format. y_real : list of tuples or list of lists of tuples - The real (actual) ranges. + The real/actual (ground truth) ranges. + - Each tuple represents a range (start, end) of the anomaly where + start is starting index (inclusive) and end is ending index (inclusive). + - If y_real is in the format of list of lists, they will be flattened into a + single list of tuples bringing it to the above format. gamma : str Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. (default: "one") @@ -152,6 +208,13 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm ------- float Global Recall + + References + ---------- + .. [1] Tatbul, Nesime, Tae Jun Lee, Stan Zdonik, Mejbah Alam,and Justin Gottschlich. + "Precision and Recall for Time Series." 32nd Conference on Neural Information + Processing Systems (NeurIPS 2018), Montréal, Canada. + http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ # Flattening y_pred and y_real flat_y_pred = _flatten_ranges(y_pred) @@ -191,12 +254,23 @@ def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm """ Calculate F1-Score for time series anomaly detection. + F-1 Score is the harmonic mean of Global Precision and Gloval recall, providing + a single metric to evaluate the performance of an anomaly detection model. + Parameters ---------- y_pred : list of tuples or list of lists of tuples - The predicted ranges. + The predicted anomaly ranges. + - Each tuple represents a range (start, end) of the anomaly where + start is starting index (inclusive) and end is ending index (inclusive). + - If y_pred is in the format of list of lists, they will be flattened into a + single list of tuples bringing it to the above format. y_real : list of tuples or list of lists of tuples - The real (actual) ranges. + The real/actual (ground truth) ranges. + - Each tuple represents a range (start, end) of the anomaly where + start is starting index (inclusive) and end is ending index (inclusive). + - If y_real is in the format of list of lists, they will be flattened into a + single list of tuples bringing it to the above format. gamma : str Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. (default: "one") @@ -210,8 +284,15 @@ def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm ------- float F1-Score + + References + ---------- + .. [1] Tatbul, Nesime, Tae Jun Lee, Stan Zdonik, Mejbah Alam,and Justin Gottschlich. + "Precision and Recall for Time Series." 32nd Conference on Neural Information + Processing Systems (NeurIPS 2018), Montréal, Canada. + http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ - precision = ts_precision(y_pred, y_real, gamma, bias_type, udf_gamma=udf_gamma) + precision = ts_precision(y_pred, y_real, bias_type) recall = ts_recall(y_pred, y_real, gamma, bias_type, alpha, udf_gamma=udf_gamma) if precision + recall > 0: diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index d30c1e873b..ad337f6886 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -54,7 +54,7 @@ ) def test_metrics(y_pred, y_real, expected_precision, expected_recall, expected_f1): """Test the range-based anomaly detection metrics.""" - precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") + precision = ts_precision(y_pred, y_real, bias_type="flat") recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) From da81823cec6f06e12f8c54e86701765e77419cb3 Mon Sep 17 00:00:00 2001 From: Aryan Date: Wed, 15 Jan 2025 16:47:18 +0530 Subject: [PATCH 13/21] Changed test cases into seperate functions --- .../anomaly_detection/tests/test_metrics.py | 302 +++++++++++++++--- 1 file changed, 251 insertions(+), 51 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index ad337f6886..c5ff29addf 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -1,7 +1,6 @@ """Test cases for the range-based anomaly detection metrics.""" import numpy as np -import pytest from aeon.benchmarking.metrics.anomaly_detection.range_metrics import ( ts_fscore, @@ -10,70 +9,271 @@ ) -# Test cases for metrics -@pytest.mark.parametrize( - "y_pred, y_real, expected_precision, expected_recall, expected_f1", - [ - ([(1, 4)], [(2, 6)], 0.750000, 0.600000, 0.666667), # Single Overlapping Range - ( - [(1, 2), (7, 8)], - [(3, 4), (9, 10)], - 0.000000, - 0.000000, - 0.000000, - ), # Multiple Non-Overlapping Ranges - ( - [(1, 3), (5, 7)], - [(2, 6), (8, 10)], - 0.666667, - 0.500000, - 0.571429, - ), # Multiple Overlapping Ranges - ( - [[(1, 3), (5, 7)], [(10, 12)]], - [(2, 6), (8, 10)], - 0.555556, - 0.625000, - 0.588235, - ), # Nested Lists of Predictions - ( - [(1, 10)], - [(2, 3), (5, 6), (8, 9)], - 0.600000, - 1.000000, - 0.750000, - ), # All Encompassing Range - ( - [(1, 2)], - [(1, 1)], - 0.5, - 1.000000, - 0.666667, - ), # Converted Binary to Range-Based(Existing example) - ], -) -def test_metrics(y_pred, y_real, expected_precision, expected_recall, expected_f1): - """Test the range-based anomaly detection metrics.""" +def test_single_overlapping_range(): + """Test for single overlapping range.""" + y_pred = [(1, 4)] + y_real = [(2, 6)] + expected_precision = 0.750000 + expected_recall = 0.600000 + expected_f1 = 0.666667 + + precision = ts_precision(y_pred, y_real, bias_type="flat") + recall = ts_recall( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + f1_score = ts_fscore( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=( + f"Precision failed for single overlapping range! " + f"Expected={expected_precision}, Got={precision}" + ), + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=( + f"Recall failed for single overlapping range! " + f"Expected={expected_recall}, Got={recall}" + ), + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=( + f"F1-Score failed for single overlapping range! " + f"Expected={expected_f1}, Got={f1_score}" + ), + ) + + +def test_multiple_non_overlapping_ranges(): + """Test for multiple non-overlapping ranges.""" + y_pred = [(1, 2), (7, 8)] + y_real = [(3, 4), (9, 10)] + expected_precision = 0.000000 + expected_recall = 0.000000 + expected_f1 = 0.000000 + + precision = ts_precision(y_pred, y_real, bias_type="flat") + recall = ts_recall( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + f1_score = ts_fscore( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=( + f"Precision failed for multiple non-overlapping ranges! " + f"Expected={expected_precision}, Got={precision}" + ), + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=( + f"Recall failed for multiple non-overlapping ranges! " + f"Expected={expected_recall}, Got={recall}" + ), + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=( + f"F1-Score failed for multiple non-overlapping ranges! " + f"Expected={expected_f1}, Got={f1_score}" + ), + ) + + +def test_multiple_overlapping_ranges(): + """Test for multiple overlapping ranges.""" + y_pred = [(1, 3), (5, 7)] + y_real = [(2, 6), (8, 10)] + expected_precision = 0.666667 + expected_recall = 0.500000 + expected_f1 = 0.571429 + + precision = ts_precision(y_pred, y_real, bias_type="flat") + recall = ts_recall( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + f1_score = ts_fscore( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=( + f"Precision failed for multiple overlapping ranges! " + f"Expected={expected_precision}, Got={precision}" + ), + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=( + f"Recall failed for multiple overlapping ranges! " + f"Expected={expected_recall}, Got={recall}" + ), + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=( + f"F1-Score failed for multiple overlapping ranges! " + f"Expected={expected_f1}, Got={f1_score}" + ), + ) + + +def test_nested_lists_of_predictions(): + """Test for nested lists of predictions.""" + y_pred = [[(1, 3), (5, 7)], [(10, 12)]] + y_real = [(2, 6), (8, 10)] + expected_precision = 0.555556 + expected_recall = 0.625000 + expected_f1 = 0.588235 + precision = ts_precision(y_pred, y_real, bias_type="flat") - recall = ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) - f1_score = ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0) + recall = ts_recall( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + f1_score = ts_fscore( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=( + f"Precision failed for nested lists of predictions! " + f"Expected={expected_precision}, Got={precision}" + ), + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=( + f"Recall failed for nested lists of predictions! " + f"Expected={expected_recall}, Got={recall}" + ), + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=( + f"F1-Score failed for nested lists of predictions! " + f"Expected={expected_f1}, Got={f1_score}" + ), + ) + + +def test_all_encompassing_range(): + """Test for all encompassing range.""" + y_pred = [(1, 10)] + y_real = [(2, 3), (5, 6), (8, 9)] + expected_precision = 0.600000 + expected_recall = 1.000000 + expected_f1 = 0.750000 + + precision = ts_precision(y_pred, y_real, bias_type="flat") + recall = ts_recall( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + f1_score = ts_fscore( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=( + f"Precision failed for all encompassing range! " + f"Expected={expected_precision}, Got={precision}" + ), + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=( + f"Recall failed for all encompassing range! " + f"Expected={expected_recall}, Got={recall}" + ), + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=( + f"F1-Score failed for all encompassing range! " + f"Expected={expected_f1}, Got={f1_score}" + ), + ) + + +def test_converted_binary_to_range_based(): + """Test for converted binary to range-based (existing example).""" + y_pred = [(1, 2)] + y_real = [(1, 1)] + expected_precision = 0.500000 + expected_recall = 1.000000 + expected_f1 = 0.666667 + + precision = ts_precision(y_pred, y_real, bias_type="flat") + recall = ts_recall( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) + f1_score = ts_fscore( + y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + ) - # Use assertions with detailed error messages for debugging np.testing.assert_almost_equal( precision, expected_precision, decimal=6, - err_msg=f"Precision failed! Expected={expected_precision}, Got={precision}", + err_msg=( + f"Precision failed for converted binary to range-based! " + f"Expected={expected_precision}, Got={precision}" + ), ) np.testing.assert_almost_equal( recall, expected_recall, decimal=6, - err_msg=f"Recall failed! Expected={expected_recall}, Got={recall}", + err_msg=( + f"Recall failed for converted binary to range-based! " + f"Expected={expected_recall}, Got={recall}" + ), ) np.testing.assert_almost_equal( f1_score, expected_f1, decimal=6, - err_msg=f"F1-Score failed! Expected={expected_f1}, Got={f1_score}", + err_msg=( + f"F1-Score failed for converted binary to range-based! " + f"Expected={expected_f1}, Got={f1_score}" + ), ) From f9732eb4f65a047ac6fe581b7f269b6405c3db48 Mon Sep 17 00:00:00 2001 From: Aryan Date: Fri, 17 Jan 2025 11:29:49 +0530 Subject: [PATCH 14/21] test cases added and added range recall --- .../anomaly_detection/range_metrics.py | 182 +++++++++++++----- .../anomaly_detection/tests/test_metrics.py | 165 +++++++++++++++- 2 files changed, 286 insertions(+), 61 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index a17e849bd6..ed0548d10a 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -52,7 +52,12 @@ def _calculate_bias(position, length, bias_type="flat"): elif bias_type == "front": return 1.0 - (position - 1) / length elif bias_type == "middle": - return 1.0 - abs(2 * (position - 1) / (length - 1) - 1) if length > 1 else 1.0 + if length / 2 == 0: + return 1.0 + if position <= length / 2: + return position / (length / 2) + else: + return (length - position + 1) / (length / 2) elif bias_type == "back": return position / length else: @@ -98,11 +103,79 @@ def _gamma_select(cardinality, gamma, udf_gamma=None): ) -def ts_precision(y_pred, y_real, bias_type="flat"): +def calculate_overlap_reward_precision(pred_range, overlap_set, bias_type): + """Overlap Reward for y_pred. + + Parameters + ---------- + pred_range : tuple + The predicted range. + overlap_set : set + The set of overlapping positions. + bias_type : str + Type of bias to apply, Should be one of ["flat", "front", "middle", "back"]. + + Returns + ------- + float + The weighted value for overlapping positions only. + """ + start, end = pred_range + length = end - start + 1 + + max_value = 0 # Total possible weighted value for all positions. + my_value = 0 # Weighted value for overlapping positions only. + + for i in range(1, length + 1): + global_position = start + i - 1 + bias_value = _calculate_bias(i, length, bias_type) + max_value += bias_value + + if global_position in overlap_set: + my_value += bias_value + + return my_value / max_value if max_value > 0 else 0.0 + + +def calculate_overlap_reward_recall(real_range, overlap_set, bias_type): + """Overlap Reward for y_real. + + Parameters + ---------- + real_range : tuple + The real range. + overlap_set : set + The set of overlapping positions. + bias_type : str + Type of bias to apply, Should be one of ["flat", "front", "middle", "back"]. + + Returns + ------- + float + The weighted value for overlapping positions only. + """ + start, end = real_range + length = end - start + 1 + + max_value = 0.0 # Total possible weighted value for all positions. + my_value = 0.0 # Weighted value for overlapping positions only. + + for i in range(1, length + 1): + global_position = start + i - 1 + bias_value = _calculate_bias(i, length, bias_type) + max_value += bias_value + + if global_position in overlap_set: + my_value += bias_value + + return my_value / max_value if max_value > 0 else 0.0 + + +def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): """ - Calculate Global Precision for time series anomaly detection. + Calculate Precision for time series anomaly detection. - Global Precision measures the proportion of correctly predicted anomaly positions + Precision measures the proportion of correctly predicted anomaly positions out of all all the predicted anomaly positions, aggregated across the entire time series. @@ -123,11 +196,16 @@ def ts_precision(y_pred, y_real, bias_type="flat"): bias_type : str Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. (default: "flat") + gamma : str + Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. + (default: "one") + udf_gamma : int or None + User-defined gamma value. (default: None) Returns ------- float - Global Precision + Precision References ---------- @@ -140,42 +218,40 @@ def ts_precision(y_pred, y_real, bias_type="flat"): flat_y_pred = _flatten_ranges(y_pred) flat_y_real = _flatten_ranges(y_real) - overlapping_weighted_positions = 0.0 - total_pred_weight = 0.0 + total_overlap_reward = 0.0 + total_cardinality = 0 for pred_range in flat_y_pred: - start_pred, end_pred = pred_range - length_pred = end_pred - start_pred + 1 + overlap_set = set() + cardinality = 0 - for i in range(1, length_pred + 1): - pos = start_pred + i - 1 - bias = _calculate_bias(i, length_pred, bias_type) + for real_start, real_end in flat_y_real: + overlap_start = max(pred_range[0], real_start) + overlap_end = min(pred_range[1], real_end) - # Check if the position is in any real range - in_real = any( - real_start <= pos <= real_end for real_start, real_end in flat_y_real - ) + if overlap_start <= overlap_end: + overlap_set.update(range(overlap_start, overlap_end + 1)) + cardinality += 1 - if in_real: - # For precision, gamma is fixed to "one" - gamma_value = 1.0 - overlapping_weighted_positions += bias * gamma_value + overlap_reward = calculate_overlap_reward_precision( + pred_range, overlap_set, bias_type + ) + gamma_value = _gamma_select(cardinality, gamma, udf_gamma) - total_pred_weight += bias + total_overlap_reward += gamma_value * overlap_reward + total_cardinality += 1 precision = ( - overlapping_weighted_positions / total_pred_weight - if total_pred_weight > 0 - else 0.0 + total_overlap_reward / total_cardinality if total_cardinality > 0 else 0.0 ) return precision def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None): """ - Calculate Global Recall for time series anomaly detection. + Calculate Recall for time series anomaly detection. - Global Recall measures the proportion of correctly predicted anomaly positions + Recall measures the proportion of correctly predicted anomaly positions out of all the real/actual (ground truth) anomaly positions, aggregated across the entire time series. @@ -207,7 +283,7 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm Returns ------- float - Global Recall + Recall References ---------- @@ -216,37 +292,41 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm Processing Systems (NeurIPS 2018), Montréal, Canada. http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ - # Flattening y_pred and y_real + # Flattening y_pred and y_real to resolve nested lists flat_y_pred = _flatten_ranges(y_pred) flat_y_real = _flatten_ranges(y_real) - overlapping_weighted_positions = 0.0 - total_real_weight = 0.0 + total_overlap_reward = 0.0 for real_range in flat_y_real: - start_real, end_real = real_range - length_real = end_real - start_real + 1 + overlap_set = set() + cardinality = 0 - for i in range(1, length_real + 1): - pos = start_real + i - 1 - bias = _calculate_bias(i, length_real, bias_type) + for pred_range in flat_y_pred: + overlap_start = max(real_range[0], pred_range[0]) + overlap_end = min(real_range[1], pred_range[1]) - # Check if the position is in any predicted range - in_pred = any( - pred_start <= pos <= pred_end for pred_start, pred_end in flat_y_pred - ) + if overlap_start <= overlap_end: + overlap_set.update(range(overlap_start, overlap_end + 1)) + cardinality += 1 - if in_pred: - gamma_value = _gamma_select(1, gamma, udf_gamma) - overlapping_weighted_positions += bias * gamma_value + # Existence Reward + existence_reward = 1.0 if overlap_set else 0.0 - total_real_weight += bias + if overlap_set: + overlap_reward = calculate_overlap_reward_recall( + real_range, overlap_set, bias_type + ) + gamma_value = _gamma_select(cardinality, gamma, udf_gamma) + overlap_reward *= gamma_value + else: + overlap_reward = 0.0 - recall = ( - overlapping_weighted_positions / total_real_weight - if total_real_weight > 0 - else 0.0 - ) + # Total Recall Score + recall_score = alpha * existence_reward + (1 - alpha) * overlap_reward + total_overlap_reward += recall_score + + recall = total_overlap_reward / len(flat_y_real) if flat_y_real else 0.0 return recall @@ -254,7 +334,7 @@ def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm """ Calculate F1-Score for time series anomaly detection. - F-1 Score is the harmonic mean of Global Precision and Gloval recall, providing + F-1 Score is the harmonic mean of Precision and Recall, providing a single metric to evaluate the performance of an anomaly detection model. Parameters @@ -292,8 +372,8 @@ def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm Processing Systems (NeurIPS 2018), Montréal, Canada. http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ - precision = ts_precision(y_pred, y_real, bias_type) - recall = ts_recall(y_pred, y_real, gamma, bias_type, alpha, udf_gamma=udf_gamma) + precision = ts_precision(y_pred, y_real, gamma, bias_type, udf_gamma) + recall = ts_recall(y_pred, y_real, gamma, bias_type, alpha, udf_gamma) if precision + recall > 0: fscore = 2 * (precision * recall) / (precision + recall) diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index c5ff29addf..40260886af 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -17,7 +17,7 @@ def test_single_overlapping_range(): expected_recall = 0.600000 expected_f1 = 0.666667 - precision = ts_precision(y_pred, y_real, bias_type="flat") + precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") recall = ts_recall( y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) @@ -62,7 +62,7 @@ def test_multiple_non_overlapping_ranges(): expected_recall = 0.000000 expected_f1 = 0.000000 - precision = ts_precision(y_pred, y_real, bias_type="flat") + precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") recall = ts_recall( y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) @@ -104,10 +104,10 @@ def test_multiple_overlapping_ranges(): y_pred = [(1, 3), (5, 7)] y_real = [(2, 6), (8, 10)] expected_precision = 0.666667 - expected_recall = 0.500000 - expected_f1 = 0.571429 + expected_recall = 0.400000 + expected_f1 = 0.500000 - precision = ts_precision(y_pred, y_real, bias_type="flat") + precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") recall = ts_recall( y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) @@ -149,10 +149,10 @@ def test_nested_lists_of_predictions(): y_pred = [[(1, 3), (5, 7)], [(10, 12)]] y_real = [(2, 6), (8, 10)] expected_precision = 0.555556 - expected_recall = 0.625000 - expected_f1 = 0.588235 + expected_recall = 0.566667 + expected_f1 = 0.561056 - precision = ts_precision(y_pred, y_real, bias_type="flat") + precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") recall = ts_recall( y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) @@ -197,7 +197,7 @@ def test_all_encompassing_range(): expected_recall = 1.000000 expected_f1 = 0.750000 - precision = ts_precision(y_pred, y_real, bias_type="flat") + precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") recall = ts_recall( y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) @@ -242,7 +242,7 @@ def test_converted_binary_to_range_based(): expected_recall = 1.000000 expected_f1 = 0.666667 - precision = ts_precision(y_pred, y_real, bias_type="flat") + precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") recall = ts_recall( y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) @@ -277,3 +277,148 @@ def test_converted_binary_to_range_based(): f"Expected={expected_f1}, Got={f1_score}" ), ) + + +def test_multiple_overlapping_ranges_with_gamma_reciprocal(): + """Test for multiple overlapping ranges with gamma=reciprocal.""" + y_pred = [(1, 3), (5, 7)] + y_real = [(2, 6), (8, 10)] + expected_precision = 0.666667 + expected_recall = 0.200000 + expected_f1 = 0.307692 + + precision = ts_precision(y_pred, y_real, gamma="reciprocal", bias_type="flat") + recall = ts_recall( + y_pred, y_real, gamma="reciprocal", bias_type="flat", alpha=0.0, udf_gamma=None + ) + f1_score = ts_fscore( + y_pred, y_real, gamma="reciprocal", bias_type="flat", alpha=0.0, udf_gamma=None + ) + + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=( + f"Precision failed for multiple overlapping ranges! " + f"Expected={expected_precision}, Got={precision}" + ), + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=( + f"Recall failed for multiple overlapping ranges! " + f"Expected={expected_recall}, Got={recall}" + ), + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=( + f"F1-Score failed for multiple overlapping ranges! " + f"Expected={expected_f1}, Got={f1_score}" + ), + ) + + +def test_multiple_overlapping_ranges_with_bias_middle(): + """Test for multiple overlapping ranges with bias_type=middle.""" + y_pred = [(1, 3), (5, 7)] + y_real = [(2, 6), (8, 10)] + expected_precision = 0.750000 + expected_recall = 0.333333 + expected_f1 = 0.461538 + + precision = ts_precision(y_pred, y_real, gamma="one", bias_type="middle") + recall = ts_recall( + y_pred, y_real, gamma="one", bias_type="middle", alpha=0.0, udf_gamma=None + ) + f1_score = ts_fscore( + y_pred, y_real, gamma="one", bias_type="middle", alpha=0.0, udf_gamma=None + ) + + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=( + f"Precision failed for multiple overlapping ranges! " + f"Expected={expected_precision}, Got={precision}" + ), + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=( + f"Recall failed for multiple overlapping ranges! " + f"Expected={expected_recall}, Got={recall}" + ), + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=( + f"F1-Score failed for multiple overlapping ranges! " + f"Expected={expected_f1}, Got={f1_score}" + ), + ) + + +def test_multiple_overlapping_ranges_with_bias_middle_gamma_reciprocal(): + """Test for multiple overlapping ranges with bias_type=middle, gamma=reciprocal.""" + y_pred = [(1, 3), (5, 7)] + y_real = [(2, 6), (8, 10)] + expected_precision = 0.750000 + expected_recall = 0.166667 + expected_f1 = 0.272727 + + precision = ts_precision(y_pred, y_real, gamma="reciprocal", bias_type="middle") + recall = ts_recall( + y_pred, + y_real, + gamma="reciprocal", + bias_type="middle", + alpha=0.0, + udf_gamma=None, + ) + f1_score = ts_fscore( + y_pred, + y_real, + gamma="reciprocal", + bias_type="middle", + alpha=0.0, + udf_gamma=None, + ) + + np.testing.assert_almost_equal( + precision, + expected_precision, + decimal=6, + err_msg=( + f"Precision failed for multiple overlapping ranges! " + f"Expected={expected_precision}, Got={precision}" + ), + ) + np.testing.assert_almost_equal( + recall, + expected_recall, + decimal=6, + err_msg=( + f"Recall failed for multiple overlapping ranges! " + f"Expected={expected_recall}, Got={recall}" + ), + ) + np.testing.assert_almost_equal( + f1_score, + expected_f1, + decimal=6, + err_msg=( + f"F1-Score failed for multiple overlapping ranges! " + f"Expected={expected_f1}, Got={f1_score}" + ), + ) From 48238f3bec284fc2822ebd8b364078042c95befd Mon Sep 17 00:00:00 2001 From: Aryan Date: Fri, 17 Jan 2025 11:34:56 +0530 Subject: [PATCH 15/21] udf_gamma removed from precision --- .../benchmarking/metrics/anomaly_detection/range_metrics.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index ed0548d10a..075191d36d 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -171,7 +171,7 @@ def calculate_overlap_reward_recall(real_range, overlap_set, bias_type): return my_value / max_value if max_value > 0 else 0.0 -def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): +def ts_precision(y_pred, y_real, gamma="one", bias_type="flat"): """ Calculate Precision for time series anomaly detection. @@ -199,8 +199,6 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): gamma : str Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. (default: "one") - udf_gamma : int or None - User-defined gamma value. (default: None) Returns ------- @@ -236,7 +234,7 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): overlap_reward = calculate_overlap_reward_precision( pred_range, overlap_set, bias_type ) - gamma_value = _gamma_select(cardinality, gamma, udf_gamma) + gamma_value = _gamma_select(cardinality, gamma) total_overlap_reward += gamma_value * overlap_reward total_cardinality += 1 From 0561981131ef6f0790f86db0b35d5a572a868c4d Mon Sep 17 00:00:00 2001 From: Aryan Date: Fri, 17 Jan 2025 12:05:54 +0530 Subject: [PATCH 16/21] changes --- .../metrics/anomaly_detection/range_metrics.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 075191d36d..78292a2c76 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -171,7 +171,7 @@ def calculate_overlap_reward_recall(real_range, overlap_set, bias_type): return my_value / max_value if max_value > 0 else 0.0 -def ts_precision(y_pred, y_real, gamma="one", bias_type="flat"): +def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): """ Calculate Precision for time series anomaly detection. @@ -205,6 +205,11 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat"): float Precision + Raises + ------ + ValueError + If an invalid `gamma` type is provided. + References ---------- .. [1] Tatbul, Nesime, Tae Jun Lee, Stan Zdonik, Mejbah Alam,and Justin Gottschlich. @@ -212,6 +217,11 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat"): Processing Systems (NeurIPS 2018), Montréal, Canada. http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ + if gamma not in ["reciprocal", "one"]: + raise ValueError("Invalid gamma type for precision. Use 'reciprocal' or 'one'.") + if udf_gamma is not None: + raise ValueError("`udf_gamma` is not applicable for precision calculations.") + # Flattening y_pred and y_real to resolve nested lists flat_y_pred = _flatten_ranges(y_pred) flat_y_real = _flatten_ranges(y_real) @@ -234,7 +244,7 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat"): overlap_reward = calculate_overlap_reward_precision( pred_range, overlap_set, bias_type ) - gamma_value = _gamma_select(cardinality, gamma) + gamma_value = _gamma_select(cardinality, gamma, udf_gamma) total_overlap_reward += gamma_value * overlap_reward total_cardinality += 1 From 4f4f617e7b5296d92b072cd5311877052c277d7f Mon Sep 17 00:00:00 2001 From: Aryan Date: Fri, 17 Jan 2025 13:03:24 +0530 Subject: [PATCH 17/21] more changes --- .../metrics/anomaly_detection/range_metrics.py | 9 ++++----- .../metrics/anomaly_detection/tests/test_metrics.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 78292a2c76..27f43259cc 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -197,7 +197,7 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. (default: "flat") gamma : str - Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. + Cardinality type. Should be one of ["reciprocal", "one"]. (default: "one") Returns @@ -217,10 +217,10 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): Processing Systems (NeurIPS 2018), Montréal, Canada. http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ - if gamma not in ["reciprocal", "one"]: - raise ValueError("Invalid gamma type for precision. Use 'reciprocal' or 'one'.") if udf_gamma is not None: raise ValueError("`udf_gamma` is not applicable for precision calculations.") + if gamma not in ["reciprocal", "one"]: + raise ValueError("Invalid gamma type for precision. Use 'reciprocal' or 'one'.") # Flattening y_pred and y_real to resolve nested lists flat_y_pred = _flatten_ranges(y_pred) @@ -244,8 +244,7 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): overlap_reward = calculate_overlap_reward_precision( pred_range, overlap_set, bias_type ) - gamma_value = _gamma_select(cardinality, gamma, udf_gamma) - + gamma_value = _gamma_select(cardinality, gamma) total_overlap_reward += gamma_value * overlap_reward total_cardinality += 1 diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index 40260886af..3ccf4f38f5 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -377,7 +377,7 @@ def test_multiple_overlapping_ranges_with_bias_middle_gamma_reciprocal(): expected_recall = 0.166667 expected_f1 = 0.272727 - precision = ts_precision(y_pred, y_real, gamma="reciprocal", bias_type="middle") + precision = ts_precision(y_pred, y_real, bias_type="middle") recall = ts_recall( y_pred, y_real, From 26b5029cd06f7f73da96aa28050ada317429522b Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 20 Jan 2025 17:57:09 +0530 Subject: [PATCH 18/21] recommended changes --- .../anomaly_detection/range_metrics.py | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 27f43259cc..6de51bb0f6 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -43,9 +43,8 @@ def _calculate_bias(position, length, bias_type="flat"): Current position in the range length : int Total length of the range - bias_type : str + bias_type : str, default="flat" Type of bias to apply, Should be one of ["flat", "front", "middle", "back"]. - (default: "flat") """ if bias_type == "flat": return 1.0 @@ -103,7 +102,7 @@ def _gamma_select(cardinality, gamma, udf_gamma=None): ) -def calculate_overlap_reward_precision(pred_range, overlap_set, bias_type): +def _calculate_overlap_reward_precision(pred_range, overlap_set, bias_type): """Overlap Reward for y_pred. Parameters @@ -137,7 +136,7 @@ def calculate_overlap_reward_precision(pred_range, overlap_set, bias_type): return my_value / max_value if max_value > 0 else 0.0 -def calculate_overlap_reward_recall(real_range, overlap_set, bias_type): +def _calculate_overlap_reward_recall(real_range, overlap_set, bias_type): """Overlap Reward for y_real. Parameters @@ -185,20 +184,16 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): The predicted anomaly ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). - - If y_pred is in the format of list of lists, they will be flattened into a \ - single list of tuples bringing it to the above format. y_real : list of tuples or list of lists of tuples The real/actual (ground truth) ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). - If y_real is in the format of list of lists, they will be flattened into a single list of tuples bringing it to the above format. - bias_type : str + bias_type : str, default="flat" Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. - (default: "flat") - gamma : str + gamma : str, default="one" Cardinality type. Should be one of ["reciprocal", "one"]. - (default: "one") Returns ------- @@ -207,6 +202,8 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): Raises ------ + ValueError + If `udf_gamma` is provided. ValueError If an invalid `gamma` type is provided. @@ -241,7 +238,7 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): overlap_set.update(range(overlap_start, overlap_end + 1)) cardinality += 1 - overlap_reward = calculate_overlap_reward_precision( + overlap_reward = _calculate_overlap_reward_precision( pred_range, overlap_set, bias_type ) gamma_value = _gamma_select(cardinality, gamma) @@ -268,24 +265,20 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm The predicted anomaly ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). - - If y_pred is in the format of list of lists, they will be flattened into a - single list of tuples bringing it to the above format. y_real : list of tuples or list of lists of tuples The real/actual (ground truth) ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). - If y_real is in the format of list of lists, they will be flattened into a single list of tuples bringing it to the above format. - gamma : str + gamma : str, default="one" Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. - (default: "one") - bias_type : str + bias_type : str, default="flat" Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. - (default: "flat") alpha : float Weight for existence reward in recall calculation. (default: 0.0) - udf_gamma : int or None - User-defined gamma value. (default: None) + udf_gamma : int or None, default=None + User-defined gamma value. Returns ------- @@ -321,7 +314,7 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm existence_reward = 1.0 if overlap_set else 0.0 if overlap_set: - overlap_reward = calculate_overlap_reward_recall( + overlap_reward = _calculate_overlap_reward_recall( real_range, overlap_set, bias_type ) gamma_value = _gamma_select(cardinality, gamma, udf_gamma) @@ -350,22 +343,18 @@ def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm The predicted anomaly ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). - - If y_pred is in the format of list of lists, they will be flattened into a - single list of tuples bringing it to the above format. y_real : list of tuples or list of lists of tuples The real/actual (ground truth) ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). - If y_real is in the format of list of lists, they will be flattened into a single list of tuples bringing it to the above format. - gamma : str + gamma : str, default="one" Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. - (default: "one") - bias_type : str + bias_type : str, default="flat" Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. - (default: "flat") - udf_gamma : int or None - User-defined gamma value. (default: None) + udf_gamma : int or None, default=None + User-defined gamma value. Returns ------- From fa60406d347005a5224a68d83e8e9be87a4de8c1 Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 20 Jan 2025 18:03:14 +0530 Subject: [PATCH 19/21] changes --- .../benchmarking/metrics/anomaly_detection/range_metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 6de51bb0f6..78fc79008c 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -180,7 +180,7 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): Parameters ---------- - y_pred : list of tuples or list of lists of tuples + y_pred : list of tuples The predicted anomaly ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). @@ -261,7 +261,7 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm Parameters ---------- - y_pred : list of tuples or list of lists of tuples + y_pred : list of tuples The predicted anomaly ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). @@ -339,7 +339,7 @@ def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm Parameters ---------- - y_pred : list of tuples or list of lists of tuples + y_pred : list of tuples The predicted anomaly ranges. - Each tuple represents a range (start, end) of the anomaly where start is starting index (inclusive) and end is ending index (inclusive). From c48d42659e49c223756a567d0a74ba9d66d21a51 Mon Sep 17 00:00:00 2001 From: Aryan Date: Thu, 23 Jan 2025 15:34:07 +0530 Subject: [PATCH 20/21] Added Parameters --- .../anomaly_detection/range_metrics.py | 31 +++++-- .../anomaly_detection/tests/test_metrics.py | 82 ++++++++++++++++--- 2 files changed, 94 insertions(+), 19 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 78fc79008c..5ca09604ef 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -275,8 +275,8 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. bias_type : str, default="flat" Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. - alpha : float - Weight for existence reward in recall calculation. (default: 0.0) + alpha : float, default: 0.0 + Weight for existence reward in recall calculation. udf_gamma : int or None, default=None User-defined gamma value. @@ -330,7 +330,16 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm return recall -def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None): +def ts_fscore( + y_pred, + y_real, + gamma="one", + p_bias="flat", + r_bias="flat", + p_alpha=0.0, + r_alpha=0.0, + udf_gamma=None, +): """ Calculate F1-Score for time series anomaly detection. @@ -351,8 +360,16 @@ def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm single list of tuples bringing it to the above format. gamma : str, default="one" Cardinality type. Should be one of ["reciprocal", "one", "udf_gamma"]. - bias_type : str, default="flat" - Type of bias to apply. Should be one of ["flat", "front", "middle", "back"]. + p_bias : str, default="flat" + Type of bias to apply for precision. + Should be one of ["flat", "front", "middle", "back"]. + r_bias : str, default="flat" + Type of bias to apply for recall. + Should be one of ["flat", "front", "middle", "back"]. + p_alpha : float, default=0.0 + Weight for existence reward in Precision calculation. + r_alpha : float, default=0.0 + Weight for existence reward in Recall calculation. udf_gamma : int or None, default=None User-defined gamma value. @@ -368,8 +385,8 @@ def ts_fscore(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm Processing Systems (NeurIPS 2018), Montréal, Canada. http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ - precision = ts_precision(y_pred, y_real, gamma, bias_type, udf_gamma) - recall = ts_recall(y_pred, y_real, gamma, bias_type, alpha, udf_gamma) + precision = ts_precision(y_pred, y_real, gamma, p_bias, udf_gamma) + recall = ts_recall(y_pred, y_real, gamma, r_bias, r_alpha, udf_gamma) if precision + recall > 0: fscore = 2 * (precision * recall) / (precision + recall) diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index 3ccf4f38f5..bacb8696b1 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -22,7 +22,14 @@ def test_single_overlapping_range(): y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) f1_score = ts_fscore( - y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + y_pred, + y_real, + gamma="one", + p_bias="flat", + r_bias="flat", + p_alpha=0.0, + r_alpha=0.0, + udf_gamma=None, ) np.testing.assert_almost_equal( @@ -67,7 +74,14 @@ def test_multiple_non_overlapping_ranges(): y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) f1_score = ts_fscore( - y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + y_pred, + y_real, + gamma="one", + p_bias="flat", + r_bias="flat", + p_alpha=0.0, + r_alpha=0.0, + udf_gamma=None, ) np.testing.assert_almost_equal( @@ -112,7 +126,14 @@ def test_multiple_overlapping_ranges(): y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) f1_score = ts_fscore( - y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + y_pred, + y_real, + gamma="one", + p_bias="flat", + r_bias="flat", + p_alpha=0.0, + r_alpha=0.0, + udf_gamma=None, ) np.testing.assert_almost_equal( @@ -157,7 +178,14 @@ def test_nested_lists_of_predictions(): y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) f1_score = ts_fscore( - y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + y_pred, + y_real, + gamma="one", + p_bias="flat", + r_bias="flat", + p_alpha=0.0, + r_alpha=0.0, + udf_gamma=None, ) np.testing.assert_almost_equal( @@ -202,7 +230,14 @@ def test_all_encompassing_range(): y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None ) f1_score = ts_fscore( - y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + y_pred, + y_real, + gamma="one", + p_bias="flat", + r_bias="flat", + p_alpha=0.0, + r_alpha=0.0, + udf_gamma=None, ) np.testing.assert_almost_equal( @@ -242,12 +277,19 @@ def test_converted_binary_to_range_based(): expected_recall = 1.000000 expected_f1 = 0.666667 - precision = ts_precision(y_pred, y_real, gamma="one", bias_type="flat") + precision = ts_precision(y_pred, y_real, gamma="reciprocal", bias_type="flat") recall = ts_recall( - y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + y_pred, y_real, gamma="reciprocal", bias_type="flat", alpha=0.0, udf_gamma=None ) f1_score = ts_fscore( - y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamma=None + y_pred, + y_real, + gamma="reciprocal", + p_bias="flat", + r_bias="flat", + p_alpha=0.0, + r_alpha=0.5, + udf_gamma=None, ) np.testing.assert_almost_equal( @@ -292,7 +334,14 @@ def test_multiple_overlapping_ranges_with_gamma_reciprocal(): y_pred, y_real, gamma="reciprocal", bias_type="flat", alpha=0.0, udf_gamma=None ) f1_score = ts_fscore( - y_pred, y_real, gamma="reciprocal", bias_type="flat", alpha=0.0, udf_gamma=None + y_pred, + y_real, + gamma="reciprocal", + p_bias="flat", + r_bias="flat", + p_alpha=0.0, + r_alpha=0.0, + udf_gamma=None, ) np.testing.assert_almost_equal( @@ -337,7 +386,14 @@ def test_multiple_overlapping_ranges_with_bias_middle(): y_pred, y_real, gamma="one", bias_type="middle", alpha=0.0, udf_gamma=None ) f1_score = ts_fscore( - y_pred, y_real, gamma="one", bias_type="middle", alpha=0.0, udf_gamma=None + y_pred, + y_real, + gamma="one", + p_bias="middle", + r_bias="middle", + p_alpha=0.0, + r_alpha=0.0, + udf_gamma=None, ) np.testing.assert_almost_equal( @@ -390,8 +446,10 @@ def test_multiple_overlapping_ranges_with_bias_middle_gamma_reciprocal(): y_pred, y_real, gamma="reciprocal", - bias_type="middle", - alpha=0.0, + p_bias="middle", + r_bias="middle", + p_alpha=0.0, + r_alpha=0.0, udf_gamma=None, ) From b13ba4a9e31b83c41426129847b2c7c32ee5bfc2 Mon Sep 17 00:00:00 2001 From: Aryan Date: Fri, 24 Jan 2025 18:11:09 +0530 Subject: [PATCH 21/21] removed udf_gamma from precision --- .../metrics/anomaly_detection/range_metrics.py | 10 ++-------- .../metrics/anomaly_detection/tests/test_metrics.py | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py index 5ca09604ef..32e6cd40ea 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/range_metrics.py @@ -170,7 +170,7 @@ def _calculate_overlap_reward_recall(real_range, overlap_set, bias_type): return my_value / max_value if max_value > 0 else 0.0 -def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): +def ts_precision(y_pred, y_real, gamma="one", bias_type="flat"): """ Calculate Precision for time series anomaly detection. @@ -202,8 +202,6 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): Raises ------ - ValueError - If `udf_gamma` is provided. ValueError If an invalid `gamma` type is provided. @@ -214,8 +212,6 @@ def ts_precision(y_pred, y_real, gamma="one", bias_type="flat", udf_gamma=None): Processing Systems (NeurIPS 2018), Montréal, Canada. http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ - if udf_gamma is not None: - raise ValueError("`udf_gamma` is not applicable for precision calculations.") if gamma not in ["reciprocal", "one"]: raise ValueError("Invalid gamma type for precision. Use 'reciprocal' or 'one'.") @@ -310,7 +306,6 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm overlap_set.update(range(overlap_start, overlap_end + 1)) cardinality += 1 - # Existence Reward existence_reward = 1.0 if overlap_set else 0.0 if overlap_set: @@ -322,7 +317,6 @@ def ts_recall(y_pred, y_real, gamma="one", bias_type="flat", alpha=0.0, udf_gamm else: overlap_reward = 0.0 - # Total Recall Score recall_score = alpha * existence_reward + (1 - alpha) * overlap_reward total_overlap_reward += recall_score @@ -385,7 +379,7 @@ def ts_fscore( Processing Systems (NeurIPS 2018), Montréal, Canada. http://papers.nips.cc/paper/7462-precision-and-recall-for-time-series.pdf """ - precision = ts_precision(y_pred, y_real, gamma, p_bias, udf_gamma) + precision = ts_precision(y_pred, y_real, gamma, p_bias) recall = ts_recall(y_pred, y_real, gamma, r_bias, r_alpha, udf_gamma) if precision + recall > 0: diff --git a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py index bacb8696b1..120d76d88d 100644 --- a/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py +++ b/aeon/benchmarking/metrics/anomaly_detection/tests/test_metrics.py @@ -433,7 +433,7 @@ def test_multiple_overlapping_ranges_with_bias_middle_gamma_reciprocal(): expected_recall = 0.166667 expected_f1 = 0.272727 - precision = ts_precision(y_pred, y_real, bias_type="middle") + precision = ts_precision(y_pred, y_real, gamma="reciprocal", bias_type="middle") recall = ts_recall( y_pred, y_real,