Character Error Rate

torchmetrics/functional/text/cer.py

@@ -0,0 +1,108 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional, Tuple, Union
+from warnings import warn
+
+import torch
+from torch import Tensor, tensor
+
+
+def _edit_distance(prediction_tokens: List[str], reference_tokens: List[str]) -> int:
+    """Standard dynamic programming algorithm to compute the edit distance.
+
+    Args:
+        prediction_tokens: A tokenized predicted sentence
+        reference_tokens: A tokenized reference sentence
+    Returns:
+        (int) Edit distance between the predicted sentence and the reference sentence
+    """
+    dp = [[0] * (len(reference_tokens) + 1) for _ in range(len(prediction_tokens) + 1)]
+    for i in range(len(prediction_tokens) + 1):
+        dp[i][0] = i
+    for j in range(len(reference_tokens) + 1):
+        dp[0][j] = j
+    for i in range(1, len(prediction_tokens) + 1):
+        for j in range(1, len(reference_tokens) + 1):
+            if prediction_tokens[i - 1] == reference_tokens[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1]
+            else:
+                dp[i][j] = min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]) + 1
+    return dp[-1][-1]
+
+
+def _cer_update(
+    predictions: Union[str, List[str]],
+    references: Union[str, List[str]],
+) -> Tuple[Tensor, Tensor]:
+    """Update the cer score with the current set of references and predictions.
+
+    Args:
+        predictions: Transcription(s) to score as a string or list of strings
+        references: Reference(s) for each speech input as a string or list of strings
+    Returns:
+        (Tensor) Number of edit operations to get from the reference to the prediction, summed over all samples
+        (Tensor) Number of character over all references
+    """
+    if isinstance(predictions, str):
+        predictions = [predictions]
+    if isinstance(references, str):
+        references = [references]
+    errors = tensor(0, dtype=torch.float)
+    total = tensor(0, dtype=torch.float)
+    for prediction, reference in zip(predictions, references):
+        prediction_tokens = prediction
+        reference_tokens = reference
+        errors += _edit_distance(prediction_tokens, reference_tokens)
+        total += len(reference_tokens)
+    return errors, total
+
+
+def _cer_compute(errors: Tensor, total: Tensor) -> Tensor:
+    """Compute the Character error rate.
+
+    Args:
+        errors: Number of edit operations to get from the reference to the prediction, summed over all samples
+        total: Number of characters over all references
+    Returns:
+        (Tensor) Character error rate
+    """
+    return errors / total
+
+
+def cer(
+    predictions: Union[str, List[str]],
+    references: Union[str, List[str]],
+    concatenate_texts: Optional[bool] = None,  # TODO: remove in v0.7
+) -> Tensor:
+    """character error rate is a common metric of the performance of an automatic speech recognition system. This
+    value indicates the percentage of characters that were incorrectly predicted. The lower the value, the better the
+    performance of the ASR system with a CER of 0 being a perfect score.
+    Args:
+        predictions: Transcription(s) to score as a string or list of strings
+        references: Reference(s) for each speech input as a string or list of strings
+        concatenate_texts: Whether to concatenate all input texts or compute CER iteratively
+            This argument is deprecated in v0.6 and it will be removed in v0.7.
+    Returns:
+        (Tensor) Character error rate
+    Examples:
+        >>> predictions = ["this is the prediction", "there is an other sample"]
+        >>> references = ["this is the reference", "there is another one"]
+        >>> cer(predictions=predictions, references=references)
+        tensor(0.3415)
+    """
+    if concatenate_texts is not None:
+        warn("`concatenate_texts` has been deprecated in v0.6 and it will be removed in v0.7", DeprecationWarning)
+    errors, total = _cer_update(predictions, references)
+    return _cer_compute(errors, total)

torchmetrics/text/cer.py

@@ -0,0 +1,109 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Callable, List, Optional, Union
+from warnings import warn
+
+import torch
+from torch import Tensor, tensor
+
+from torchmetrics.functional.text.cer import _cer_compute, _cer_update
+from torchmetrics.metric import Metric
+
+
+class CER(Metric):
+    r"""
+    Character error rate (CER_) is a  metric of the performance of an automatic speech recognition system.
+    This value indicates the percentage of characters that were incorrectly predicted.
+    The lower the value, the better the performance of the ASR system with a WER of 0 being a perfect score.
+    Character error rate can then be computed as:
+
+    .. math::
+        CER = \frac{S + D + I}{N} = \frac{S + D + I}{S + D + C}
+
+    where:
+        - S is the number of substitutions,
+        - D is the number of deletions,
+        - I is the number of insertions,
+        - C is the number of correct characters,
+        - N is the number of characters in the reference (N=S+D+C).
+
+    Compute CER score of transcribed segments against references.
+
+    Args:
+        concatenate_texts: Whether to concatenate all input texts or compute WER iteratively.
+            This argument is deprecated in v0.6 and it will be removed in v0.7.
+        compute_on_step:
+            Forward only calls ``update()`` and return None if this is set to False. default: True
+        dist_sync_on_step:
+            Synchronize metric state across processes at each ``forward()``
+            before returning the value at the step. default: False
+        process_group:
+            Specify the process group on which synchronization is called. default: None (which selects the entire world)
+        dist_sync_fn:
+            Callback that performs the allgather operation on the metric state. When ``None``, DDP
+            will be used to perform the allgather
+
+    Returns:
+        (Tensor) Character error rate
+
+    Examples:
+        >>> predictions = ["this is the prediction", "there is an other sample"]
+        >>> references = ["this is the reference", "there is another one"]
+        >>> metric = CER()
+        >>> metric(predictions, references)
+        tensor(0.3415)
+    """
+    is_differentiable = False
+    higher_is_better = False
+    error: Tensor
+    total: Tensor
+
+    def __init__(
+        self,
+        concatenate_texts: Optional[bool] = None,  # TODO: remove in v0.7
+        compute_on_step: bool = True,
+        dist_sync_on_step: bool = False,
+        process_group: Optional[Any] = None,
+        dist_sync_fn: Callable = None,
+    ):
+        super().__init__(
+            compute_on_step=compute_on_step,
+            dist_sync_on_step=dist_sync_on_step,
+            process_group=process_group,
+            dist_sync_fn=dist_sync_fn,
+        )
+        if concatenate_texts is not None:
+            warn("`concatenate_texts` has been deprecated in v0.6 and it will be removed in v0.7", DeprecationWarning)
+        self.add_state("errors", tensor(0, dtype=torch.float), dist_reduce_fx="sum")
+        self.add_state("total", tensor(0, dtype=torch.float), dist_reduce_fx="sum")
+
+    def update(self, predictions: Union[str, List[str]], references: Union[str, List[str]]) -> None:  # type: ignore
+        """Store references/predictions for computing Character Error Rate scores.
+
+        Args:
+            predictions: Transcription(s) to score as a string or list of strings
+            references: Reference(s) for each speech input as a string or list of strings
+        """
+        errors, total = _cer_update(predictions, references)
+        self.errors += errors
+        self.total += total
+
+    def compute(self) -> Tensor:
+        """Calculate the character error rate.
+
+        Returns:
+            (Tensor) Character error rate
+        """
+        return _cer_compute(self.errors, self.total)