From 48d8223bda821e23359306a2424d8f37fc8d1b4e Mon Sep 17 00:00:00 2001 From: Ankit Avinash Date: Wed, 18 Oct 2023 19:49:21 +0530 Subject: [PATCH 1/7] Added Binary Focal Cross Entropy --- .../binary_focal_cross_entropy.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 machine_learning/loss_functions/binary_focal_cross_entropy.py diff --git a/machine_learning/loss_functions/binary_focal_cross_entropy.py b/machine_learning/loss_functions/binary_focal_cross_entropy.py new file mode 100644 index 000000000000..33210f4a3bc7 --- /dev/null +++ b/machine_learning/loss_functions/binary_focal_cross_entropy.py @@ -0,0 +1,64 @@ +""" +Binary Focal Cross-Entropy (BFCE) Loss Function + +Description: +Quantifies dissimilarity between true labels (0 or 1) and predicted probabilities. +It's a variation of binary cross-entropy that addresses class imbalance by +focusing on hard examples. + +Formula: +Focal Loss = -Σ(alpha * (1 - y_pred)**gamma * y_true * log(y_pred) + (1 - alpha) * y_pred**gamma * (1 - y_true) * log(1 - y_pred)) + +Source: +[Lin et al., 2018](https://arxiv.org/pdf/1708.02002.pdf) +""" + + +import numpy as np + +def binary_focal_cross_entropy( + y_true: np.ndarray, y_pred: np.ndarray, + gamma: float = 2.0, alpha: float = 0.25, epsilon: float = 1e-15 +) -> float: + """ + Calculate the BFCE Loss between true labels and predicted probabilities. + + Parameters: + - y_true: True binary labels (0 or 1). + - y_pred: Predicted probabilities for class 1. + - gamma: Focusing parameter for modulating the loss (default: 2.0). + - alpha: Weighting factor for class 1 (default: 0.25). + - epsilon: Small constant to avoid numerical instability. + + Returns: + - bcfe_loss: Binary Focal Cross-Entropy Loss. + + Example Usage: + >>> true_labels = np.array([0, 1, 1, 0, 1]) + >>> predicted_probs = np.array([0.2, 0.7, 0.9, 0.3, 0.8]) + >>> binary_focal_cross_entropy(true_labels, predicted_probs) + 0.008257977659239775 + >>> true_labels = np.array([0, 1, 1, 0, 1]) + >>> predicted_probs = np.array([0.3, 0.8, 0.9, 0.2]) + >>> binary_focal_cross_entropy(true_labels, predicted_probs) + Traceback (most recent call last): + ... + ValueError: Input arrays must have the same length. + """ + if len(y_true) != len(y_pred): + raise ValueError("Input arrays must have the same length.") + # Clip predicted probabilities to avoid log(0) and log(1) + y_pred = np.clip(y_pred, epsilon, 1 - epsilon) + + # Focal loss calculation + bcfe_loss = -(alpha * (1 - y_pred) ** gamma * y_true * np.log(y_pred) + + (1 - alpha) * y_pred ** gamma * (1 - y_true) * np.log(1 - y_pred)) + + # Take the mean over all samples + return np.mean(bcfe_loss) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() From f2cfd1f9a0bb7005ab09a76f1eb543fe94dd26ce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:21:40 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../loss_functions/binary_focal_cross_entropy.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/machine_learning/loss_functions/binary_focal_cross_entropy.py b/machine_learning/loss_functions/binary_focal_cross_entropy.py index 33210f4a3bc7..5ff4cd350534 100644 --- a/machine_learning/loss_functions/binary_focal_cross_entropy.py +++ b/machine_learning/loss_functions/binary_focal_cross_entropy.py @@ -3,7 +3,7 @@ Description: Quantifies dissimilarity between true labels (0 or 1) and predicted probabilities. -It's a variation of binary cross-entropy that addresses class imbalance by +It's a variation of binary cross-entropy that addresses class imbalance by focusing on hard examples. Formula: @@ -16,9 +16,13 @@ import numpy as np + def binary_focal_cross_entropy( - y_true: np.ndarray, y_pred: np.ndarray, - gamma: float = 2.0, alpha: float = 0.25, epsilon: float = 1e-15 + y_true: np.ndarray, + y_pred: np.ndarray, + gamma: float = 2.0, + alpha: float = 0.25, + epsilon: float = 1e-15, ) -> float: """ Calculate the BFCE Loss between true labels and predicted probabilities. @@ -51,8 +55,10 @@ def binary_focal_cross_entropy( y_pred = np.clip(y_pred, epsilon, 1 - epsilon) # Focal loss calculation - bcfe_loss = -(alpha * (1 - y_pred) ** gamma * y_true * np.log(y_pred) - + (1 - alpha) * y_pred ** gamma * (1 - y_true) * np.log(1 - y_pred)) + bcfe_loss = -( + alpha * (1 - y_pred) ** gamma * y_true * np.log(y_pred) + + (1 - alpha) * y_pred**gamma * (1 - y_true) * np.log(1 - y_pred) + ) # Take the mean over all samples return np.mean(bcfe_loss) From 13ba251e97108f09831ce20ad3b81e267d61fdad Mon Sep 17 00:00:00 2001 From: Ankit Avinash Date: Wed, 18 Oct 2023 19:53:53 +0530 Subject: [PATCH 3/7] Fixed Issue --- machine_learning/loss_functions/binary_focal_cross_entropy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/machine_learning/loss_functions/binary_focal_cross_entropy.py b/machine_learning/loss_functions/binary_focal_cross_entropy.py index 33210f4a3bc7..c94aedfcde19 100644 --- a/machine_learning/loss_functions/binary_focal_cross_entropy.py +++ b/machine_learning/loss_functions/binary_focal_cross_entropy.py @@ -7,7 +7,8 @@ focusing on hard examples. Formula: -Focal Loss = -Σ(alpha * (1 - y_pred)**gamma * y_true * log(y_pred) + (1 - alpha) * y_pred**gamma * (1 - y_true) * log(1 - y_pred)) +Focal Loss = -Σ(alpha * (1 - y_pred)**gamma * y_true * log(y_pred) + + (1 - alpha) * y_pred**gamma * (1 - y_true) * log(1 - y_pred)) Source: [Lin et al., 2018](https://arxiv.org/pdf/1708.02002.pdf) From 8d8c33f9cc16f3fe33b59b4595fe72e973bde2f5 Mon Sep 17 00:00:00 2001 From: Ankit Avinash Date: Wed, 18 Oct 2023 19:55:21 +0530 Subject: [PATCH 4/7] Fixed Issue --- machine_learning/loss_functions/binary_focal_cross_entropy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine_learning/loss_functions/binary_focal_cross_entropy.py b/machine_learning/loss_functions/binary_focal_cross_entropy.py index 7e913f7f9504..18eb05a5f432 100644 --- a/machine_learning/loss_functions/binary_focal_cross_entropy.py +++ b/machine_learning/loss_functions/binary_focal_cross_entropy.py @@ -7,7 +7,7 @@ focusing on hard examples. Formula: -Focal Loss = -Σ(alpha * (1 - y_pred)**gamma * y_true * log(y_pred) +Focal Loss = -Σ(alpha * (1 - y_pred)**gamma * y_true * log(y_pred) + (1 - alpha) * y_pred**gamma * (1 - y_true) * log(1 - y_pred)) Source: From 54cb149be0af9415becf5270a661e0dc20e5334f Mon Sep 17 00:00:00 2001 From: Ankit Avinash Date: Mon, 23 Oct 2023 09:49:21 +0530 Subject: [PATCH 5/7] Added BFCE loss to loss_functions.py --- machine_learning/loss_functions.py | 53 ++++++++++++++ .../binary_focal_cross_entropy.py | 71 ------------------- 2 files changed, 53 insertions(+), 71 deletions(-) delete mode 100644 machine_learning/loss_functions/binary_focal_cross_entropy.py diff --git a/machine_learning/loss_functions.py b/machine_learning/loss_functions.py index 0fa0956ed572..8f4ac5b17b98 100644 --- a/machine_learning/loss_functions.py +++ b/machine_learning/loss_functions.py @@ -39,6 +39,59 @@ def binary_cross_entropy( return np.mean(bce_loss) +def binary_focal_cross_entropy( + y_true: np.ndarray, + y_pred: np.ndarray, + gamma: float = 2.0, + alpha: float = 0.25, + epsilon: float = 1e-15, +) -> float: + """ + Calculate the mean binary focal cross-entropy (BFCE) loss between true labels + and predicted probabilities. + + BFCE loss quantifies dissimilarity between true labels (0 or 1) and predicted + probabilities. It's a variation of binary cross-entropy that addresses class + imbalance by focusing on hard examples. + + BCFE = -Σ(alpha * (1 - y_pred)**gamma * y_true * log(y_pred) + + (1 - alpha) * y_pred**gamma * (1 - y_true) * log(1 - y_pred)) + + Reference: [Lin et al., 2018](https://arxiv.org/pdf/1708.02002.pdf) + + Parameters: + - y_true: True binary labels (0 or 1). + - y_pred: Predicted probabilities for class 1. + - gamma: Focusing parameter for modulating the loss (default: 2.0). + - alpha: Weighting factor for class 1 (default: 0.25). + - epsilon: Small constant to avoid numerical instability. + + >>> true_labels = np.array([0, 1, 1, 0, 1]) + >>> predicted_probs = np.array([0.2, 0.7, 0.9, 0.3, 0.8]) + >>> binary_focal_cross_entropy(true_labels, predicted_probs) + 0.008257977659239775 + >>> true_labels = np.array([0, 1, 1, 0, 1]) + >>> predicted_probs = np.array([0.3, 0.8, 0.9, 0.2]) + >>> binary_focal_cross_entropy(true_labels, predicted_probs) + Traceback (most recent call last): + ... + ValueError: Input arrays must have the same length. + """ + if len(y_true) != len(y_pred): + raise ValueError("Input arrays must have the same length.") + # Clip predicted probabilities to avoid log(0) and log(1) + y_pred = np.clip(y_pred, epsilon, 1 - epsilon) + + # Focal loss calculation + bcfe_loss = -( + alpha * (1 - y_pred)**gamma * y_true * np.log(y_pred) + + (1 - alpha) * y_pred**gamma * (1 - y_true) * np.log(1 - y_pred) + ) + + # Take the mean over all samples + return np.mean(bcfe_loss) + + def categorical_cross_entropy( y_true: np.ndarray, y_pred: np.ndarray, epsilon: float = 1e-15 ) -> float: diff --git a/machine_learning/loss_functions/binary_focal_cross_entropy.py b/machine_learning/loss_functions/binary_focal_cross_entropy.py deleted file mode 100644 index 18eb05a5f432..000000000000 --- a/machine_learning/loss_functions/binary_focal_cross_entropy.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Binary Focal Cross-Entropy (BFCE) Loss Function - -Description: -Quantifies dissimilarity between true labels (0 or 1) and predicted probabilities. -It's a variation of binary cross-entropy that addresses class imbalance by -focusing on hard examples. - -Formula: -Focal Loss = -Σ(alpha * (1 - y_pred)**gamma * y_true * log(y_pred) - + (1 - alpha) * y_pred**gamma * (1 - y_true) * log(1 - y_pred)) - -Source: -[Lin et al., 2018](https://arxiv.org/pdf/1708.02002.pdf) -""" - - -import numpy as np - - -def binary_focal_cross_entropy( - y_true: np.ndarray, - y_pred: np.ndarray, - gamma: float = 2.0, - alpha: float = 0.25, - epsilon: float = 1e-15, -) -> float: - """ - Calculate the BFCE Loss between true labels and predicted probabilities. - - Parameters: - - y_true: True binary labels (0 or 1). - - y_pred: Predicted probabilities for class 1. - - gamma: Focusing parameter for modulating the loss (default: 2.0). - - alpha: Weighting factor for class 1 (default: 0.25). - - epsilon: Small constant to avoid numerical instability. - - Returns: - - bcfe_loss: Binary Focal Cross-Entropy Loss. - - Example Usage: - >>> true_labels = np.array([0, 1, 1, 0, 1]) - >>> predicted_probs = np.array([0.2, 0.7, 0.9, 0.3, 0.8]) - >>> binary_focal_cross_entropy(true_labels, predicted_probs) - 0.008257977659239775 - >>> true_labels = np.array([0, 1, 1, 0, 1]) - >>> predicted_probs = np.array([0.3, 0.8, 0.9, 0.2]) - >>> binary_focal_cross_entropy(true_labels, predicted_probs) - Traceback (most recent call last): - ... - ValueError: Input arrays must have the same length. - """ - if len(y_true) != len(y_pred): - raise ValueError("Input arrays must have the same length.") - # Clip predicted probabilities to avoid log(0) and log(1) - y_pred = np.clip(y_pred, epsilon, 1 - epsilon) - - # Focal loss calculation - bcfe_loss = -( - alpha * (1 - y_pred) ** gamma * y_true * np.log(y_pred) - + (1 - alpha) * y_pred**gamma * (1 - y_true) * np.log(1 - y_pred) - ) - - # Take the mean over all samples - return np.mean(bcfe_loss) - - -if __name__ == "__main__": - import doctest - - doctest.testmod() From 9ea3174c04fffa9bd95c8b8f5913951ceb75227d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 04:20:04 +0000 Subject: [PATCH 6/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/loss_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine_learning/loss_functions.py b/machine_learning/loss_functions.py index 8f4ac5b17b98..482619ffaa8f 100644 --- a/machine_learning/loss_functions.py +++ b/machine_learning/loss_functions.py @@ -84,7 +84,7 @@ def binary_focal_cross_entropy( # Focal loss calculation bcfe_loss = -( - alpha * (1 - y_pred)**gamma * y_true * np.log(y_pred) + alpha * (1 - y_pred) ** gamma * y_true * np.log(y_pred) + (1 - alpha) * y_pred**gamma * (1 - y_true) * np.log(1 - y_pred) ) From 2438ac2de6800f68a2513d0ca15ec37de8132b18 Mon Sep 17 00:00:00 2001 From: Tianyi Zheng Date: Mon, 23 Oct 2023 01:21:09 -0400 Subject: [PATCH 7/7] Update machine_learning/loss_functions.py --- machine_learning/loss_functions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/machine_learning/loss_functions.py b/machine_learning/loss_functions.py index 482619ffaa8f..ef34296360e2 100644 --- a/machine_learning/loss_functions.py +++ b/machine_learning/loss_functions.py @@ -79,16 +79,14 @@ def binary_focal_cross_entropy( """ if len(y_true) != len(y_pred): raise ValueError("Input arrays must have the same length.") - # Clip predicted probabilities to avoid log(0) and log(1) + # Clip predicted probabilities to avoid log(0) y_pred = np.clip(y_pred, epsilon, 1 - epsilon) - # Focal loss calculation bcfe_loss = -( alpha * (1 - y_pred) ** gamma * y_true * np.log(y_pred) + (1 - alpha) * y_pred**gamma * (1 - y_true) * np.log(1 - y_pred) ) - # Take the mean over all samples return np.mean(bcfe_loss)