[tn] english tn, support fraction (#209)

tn/english/normalizer.py

@@ -17,6 +17,7 @@
 from tn.english.rules.cardinal import Cardinal
 from tn.english.rules.ordinal import Ordinal
 from tn.english.rules.decimal import Decimal
+from tn.english.rules.fraction import Fraction
 from tn.english.rules.word import Word
 from tn.english.rules.date import Date
 
@@ -36,19 +37,22 @@ def build_tagger(self):
         cardinal = add_weight(Cardinal().tagger, 1.0)
         ordinal = add_weight(Ordinal().tagger, 1.0)
         decimal = add_weight(Decimal().tagger, 1.0)
+        fraction = add_weight(Fraction().tagger, 1.0)
         date = add_weight(Date().tagger, 0.99)
         word = add_weight(Word().tagger, 100)
         tagger = (cardinal | ordinal | word
-                  | date | decimal).optimize() + self.DELETE_SPACE
+                  | date | decimal | fraction).optimize() + self.DELETE_SPACE
         # delete the last space
         self.tagger = tagger.star @ self.build_rule(delete(' '), r='[EOS]')
 
     def build_verbalizer(self):
         cardinal = Cardinal().verbalizer
         ordinal = Ordinal().verbalizer
         decimal = Decimal().verbalizer
+        fraction = Fraction().verbalizer
         word = Word().verbalizer
         date = Date().verbalizer
         verbalizer = (cardinal | ordinal | word
-                      | date | decimal).optimize() + self.INSERT_SPACE
+                      | date | decimal
+                      | fraction).optimize() + self.INSERT_SPACE
         self.verbalizer = verbalizer.star

tn/english/rules/fraction.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2024, WENET COMMUNITY.  Xingchen Song (sxc19@tsinghua.org.cn).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.examples import plurals
+from pynini.lib import pynutil
+
+from tn.processor import Processor
+from tn.utils import get_abs_path
+from tn.english.rules.cardinal import Cardinal
+from tn.english.rules.ordinal import Ordinal
+
+
+class Fraction(Processor):
+
+    def __init__(self, deterministic: bool = False):
+        """
+        Args:
+            deterministic: if True will provide a single transduction option,
+                for False multiple transduction are generated (used for audio-based normalization)
+        """
+        super().__init__('fraction', ordertype="en_tn")
+        self.deterministic = deterministic
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        """
+        Finite state transducer for classifying fraction
+        "23 4/5" ->
+        fraction { integer_part: "twenty three" numerator: "four" denominator: "five" }
+        "23 4/5th" ->
+        fraction { integer_part: "twenty three" numerator: "four" denominator: "five" }
+        """
+        cardinal_graph = Cardinal(self.deterministic).graph
+        integer = pynutil.insert(
+            "integer_part: \"") + cardinal_graph + pynutil.insert("\"")
+        numerator = (pynutil.insert("numerator: \"") + cardinal_graph +
+                     (pynini.cross("/", "\" ") | pynini.cross(" / ", "\" ")))
+
+        endings = ["rd", "th", "st", "nd"]
+        endings += [x.upper() for x in endings]
+        optional_end = pynini.closure(pynini.cross(pynini.union(*endings), ""),
+                                      0, 1)
+
+        denominator = pynutil.insert(
+            "denominator: \""
+        ) + cardinal_graph + optional_end + pynutil.insert("\"")
+
+        graph = pynini.closure(integer + pynini.accep(" "), 0,
+                               1) + (numerator + denominator)
+        graph |= pynini.closure(
+            integer +
+            (pynini.accep(" ") | pynutil.insert(" ")), 0, 1) + pynini.compose(
+                pynini.string_file(
+                    get_abs_path("english/data/number/fraction.tsv")),
+                (numerator + denominator))
+
+        self.graph = graph
+        final_graph = self.add_tokens(self.graph)
+        self.tagger = final_graph.optimize()
+
+    def build_verbalizer(self):
+        """
+        Finite state transducer for verbalizing fraction
+            e.g. fraction { integer_part: "twenty three" numerator: "four" denominator: "five" } ->
+            twenty three and four fifth
+        """
+        suffix = Ordinal(self.deterministic).suffix
+
+        integer = pynutil.delete("integer_part: \"") + pynini.closure(
+            self.NOT_QUOTE) + pynutil.delete("\" ")
+        denominator_one = pynini.cross("denominator: \"one\"", "over one")
+        denominator_half = pynini.cross("denominator: \"two\"", "half")
+        denominator_quarter = pynini.cross("denominator: \"four\"", "quarter")
+
+        denominator_rest = (pynutil.delete("denominator: \"") +
+                            pynini.closure(self.NOT_QUOTE) @ suffix +
+                            pynutil.delete("\""))
+
+        denominators = plurals._priority_union(
+            denominator_one,
+            plurals._priority_union(
+                denominator_half,
+                plurals._priority_union(denominator_quarter, denominator_rest,
+                                        pynini.closure(self.VCHAR)),
+                pynini.closure(self.VCHAR),
+            ),
+            pynini.closure(self.VCHAR),
+        ).optimize()
+        if not self.deterministic:
+            denominators |= pynutil.delete("denominator: \"") + (
+                pynini.accep("four") @ suffix) + pynutil.delete("\"")
+
+        numerator_one = pynutil.delete("numerator: \"") + pynini.accep(
+            "one") + pynutil.delete("\" ")
+        numerator_one = numerator_one + self.INSERT_SPACE + denominators
+        numerator_rest = (
+            pynutil.delete("numerator: \"") +
+            (pynini.closure(self.NOT_QUOTE) - pynini.accep("one")) +
+            pynutil.delete("\" "))
+        numerator_rest = numerator_rest + self.INSERT_SPACE + denominators
+        numerator_rest @= pynini.cdrewrite(
+            plurals._priority_union(pynini.cross("half", "halves"),
+                                    pynutil.insert("s"),
+                                    pynini.closure(self.VCHAR)),
+            "",
+            "[EOS]",
+            pynini.closure(self.VCHAR),
+        )
+
+        graph = numerator_one | numerator_rest
+
+        conjunction = pynutil.insert("and ")
+
+        integer = pynini.closure(integer + self.INSERT_SPACE + conjunction, 0,
+                                 1)
+
+        graph = integer + graph
+        graph @= pynini.cdrewrite(
+            pynini.cross("and one half", "and a half")
+            | pynini.cross("over ones", "over one"), "", "[EOS]",
+            pynini.closure(self.VCHAR))
+
+        self.graph = graph
+        delete_tokens = self.delete_tokens(self.graph)
+        self.verbalizer = delete_tokens.optimize()

tn/english/test/data/fraction.txt

@@ -0,0 +1,7 @@
+23 4/5 => twenty three and four fifths
+23 4/5th => twenty three and four fifths
+1/3 => one third
+1/2 => one half
+1/4 => one quarter
+2/4 => two quarters
+23/44 => twenty three forty fourths

tn/english/test/fraction_test.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2024 Xingchen Song (sxc19@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from tn.english.rules.fraction import Fraction
+from tn.english.test.utils import parse_test_case
+
+
+class TestFraction:
+
+    fraction = Fraction(deterministic=False)
+    fraction_cases = parse_test_case('data/fraction.txt')
+
+    @pytest.mark.parametrize("written, spoken", fraction_cases)
+    def test_fraction(self, written, spoken):
+        assert self.fraction.normalize(written) == spoken