diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..ba38def21 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { } environment { AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0' - DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0' + DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-26-0' EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1' ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0' ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0' diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv new file mode 100644 index 000000000..d440115cf --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv @@ -0,0 +1,13 @@ +eine 1 +eins 1 +ein 1 +zwei 2 +zwo 2 +zwö 2 +drei 3 +vier 4 +fünf 5 +sechs 6 +sieben 7 +acht 8 +neun 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv new file mode 100644 index 000000000..520919081 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv @@ -0,0 +1,3 @@ +zehn 10 +elf 11 +zwölf 12 diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/lexicon.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/lexicon.tsv new file mode 100644 index 000000000..266e1038b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/lexicon.tsv @@ -0,0 +1,20 @@ +und und +minus minus +hundert hundert +ein hundert hundert +tausend tausend +million million +millionen million +milliarde milliarde +milliarden milliarde +milliard milliarde +billion billion +billionen billion +billiarde billiarde +billiarden billiarde +billiard billiarde +trillion trillion +trillionen trillion +trilliarde trilliarde +trilliarden trilliarde +trilliard trilliarde diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv new file mode 100644 index 000000000..854712ea6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv @@ -0,0 +1,10 @@ +zehn 10 +elf 11 +zwölf 12 +dreizehn 13 +vierzehn 14 +fünfzehn 15 +sechzehn 16 +siebzehn 17 +achtzehn 18 +neunzehn 19 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv new file mode 100644 index 000000000..28a3f8032 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv @@ -0,0 +1,8 @@ +zwanzig 2 +dreißig 3 +vierzig 4 +fünfzig 5 +sechzig 6 +siebzig 7 +achtzig 8 +neunzig 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv new file mode 100644 index 000000000..973c00968 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv @@ -0,0 +1 @@ +null 0 diff --git a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py new file mode 100644 index 000000000..8dc9c6391 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py @@ -0,0 +1,319 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), + NEMO_SIGMA, +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logger.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 46fdca4e3..594ad1089 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -12,10 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. + import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.inverse_text_normalization.de.graph_utils import NEMO_DIGIT, NEMO_SIGMA, NEMO_SPACE, GraphFst +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path + + +def _digit_tie_flips(): + """Map concatenated ones+tens (12 for einundzwanzig) to the written number (21).""" + return pynini.string_map([(f"{ones}{tens}", f"{tens}{ones}") for tens in range(2, 10) for ones in range(1, 10)]) + + +def _forms(lexicon, lemma): + """Spoken forms for one lemma (e.g. million / millionen).""" + return pynini.project(lexicon @ pynini.accep(lemma), "input").optimize() class CardinalFst(GraphFst): @@ -24,45 +36,275 @@ class CardinalFst(GraphFst): Allows both compound numeral strings or separated by whitespace. "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. - e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } } - e.g. minus dreiundzwanzig -> cardinal { integer: "23" } } - e.g. dreizehn -> cardinal { integer: "13" } } - e.g. ein hundert -> cardinal { integer: "100" } } - e.g. einhundert -> cardinal { integer: "100" } } - e.g. ein tausend -> cardinal { integer: "1000" } } - e.g. eintausend -> cardinal { integer: "1000" } } - e.g. ein tausend zwanzig -> cardinal { integer: "1020" } } - - Args: - tn_cardinal_tagger: TN cardinal tagger + e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } + e.g. minus dreiundzwanzig -> cardinal { negative: "-" integer: "23" } + e.g. dreizehn -> cardinal { integer: "13" } + e.g. ein hundert -> cardinal { integer: "100" } + e.g. einhundert -> cardinal { integer: "100" } + e.g. ein tausend -> cardinal { integer: "1.000" } + e.g. eintausend -> cardinal { integer: "1.000" } + e.g. ein tausend zwanzig -> cardinal { integer: "1.020" } + e.g. minus eine billion fünfundsechzig milliarden vier millionen sechs -> cardinal { negative: "-" integer: "1.065.004.000.006" } """ - def __init__(self, tn_cardinal_tagger: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + def __init__(self): + super().__init__(name="cardinal", kind="classify") + + # WFST mappings for numbers 0-99 + zero = pynini.string_file(get_abs_path("data/cardinal/zero.tsv")) + digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) + # Isolates single digit cardinals to pass to other graphs + self.digits = digits.optimize() + irregular_teens = pynini.string_file(get_abs_path("data/cardinal/irregular_teens.tsv")) + # 0-12 stay as words: zero + digits (1-9) + irregular teens (10-12) + to_denormalize = zero | digits | irregular_teens + # Isolates the first dozen + self.dozen = to_denormalize.optimize() + teens = pynini.string_file(get_abs_path("data/cardinal/teens.tsv")) + tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) + # Standalone decades: tens digit (2) + 0 -> 20 + ties = tens + pynutil.insert("0") + # German flips ones and tens in two-digit numbers (ein + zwanzig -> 21). + flips = _digit_tie_flips() + lexicon = pynini.string_file(get_abs_path("data/cardinal/lexicon.tsv")) + und = _forms(lexicon, "und") + minus = _forms(lexicon, "minus") + hundert = _forms(lexicon, "hundert") + tausend = _forms(lexicon, "tausend") + million = _forms(lexicon, "million") + milliarde = _forms(lexicon, "milliarde") + billion_de = _forms(lexicon, "billion") + billiarde = _forms(lexicon, "billiarde") + trillion_de = _forms(lexicon, "trillion") + trilliarde = _forms(lexicon, "trilliarde") + delete_space = pynutil.delete(NEMO_SPACE) + delete_und = pynutil.delete(und) + + # Accepts normalized digits+ties (ein+und+zwanzig) + digit_ties = digits + delete_space.ques + delete_und + delete_space.ques + tens + # Flips ties and digits for denormalization + ties_digit = digit_ties @ flips + + # WFST grammar for hundreds + graph_10_99 = teens | ties | ties_digit + self.graph_double_digits = graph_10_99 + # Isolates single and double-digit cardinals to pass to other graphs + graph_single_and_double_digits = digits | graph_10_99 + self.graph_single_and_double_digits = graph_single_and_double_digits.optimize() + + hundreds = (pynini.cross(hundert, "100")) | ( + ( + (digits | pynutil.insert("1")) + + delete_space.ques + + pynutil.delete(hundert) + + delete_space.ques + + delete_und.ques + + delete_space.ques + + graph_10_99 + ) + | ( + (digits | pynutil.insert("1")) + + delete_space.ques + + pynini.cross(hundert, "0") + + delete_space.ques + + delete_und.ques + + delete_space.ques + + digits + ) + | ((digits | pynutil.insert("1")) + delete_space.ques + pynini.cross(hundert, "00")) + ) + + # Digits are grouped in clusters of three: {hundreds}{tens}{ones}. + # Clusters of three are separated by periods, applied right to left. + digit_cluster = ( + (hundreds) + | (pynutil.insert("0") + graph_10_99) + | (pynutil.insert("00") + digits) + | (pynutil.insert("000")) + ) + # The subgraph below introduces three-digit clusters containing at least one non-zero digit. + # It is mainly utilized by the "years" subgraph in the DATE class. + non_zero_digit_cluster = (hundreds) | (pynutil.insert("0") + graph_10_99) | (pynutil.insert("00") + digits) + + # WFST grammar for thousands + thousands = (pynini.cross(tausend, "1.000")) | ( + ( + (pynini.cross(tausend, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(tausend, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + digit_cluster + ) + + non_zero_thousands = (pynini.cross(tausend, "1.000")) | ( + ( + (pynini.cross(tausend, "1.") + delete_space.ques + delete_und.ques) + | (non_zero_digit_cluster + delete_space.ques + pynini.cross(tausend, ".") + delete_und.ques) + # | pynutil.insert("000.") + ) + + delete_space.ques + + digit_cluster + ) + + # WFST grammar for millions + millions = (pynini.cross(million, "1.000.000")) | ( + ( + (pynini.cross(million, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(million, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + thousands + ) + + # WFST grammar for billions + billion = milliarde + billions = (pynini.cross(milliarde, "1.000.000.000")) | ( + ( + (pynini.cross(milliarde, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(billion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + millions + ) + + # WFST grammar for trillions + trillion = billion_de + trillions = (pynini.cross(billion_de, "1.000.000.000.000")) | ( + ( + (pynini.cross(billion_de, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(trillion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + billions + ) + + # WFST grammar for quadrillions + quadrillion = billiarde + quadrillions = (pynini.cross(billiarde, "1.000.000.000.000.000")) | ( + ( + (pynini.cross(quadrillion, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(quadrillion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + trillions + ) + + # WFST grammar for quintillions + quintillion = trillion_de + quintillions = (pynini.cross(trillion_de, "1.000.000.000.000.000.000")) | ( + ( + (pynini.cross(trillion_de, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(quintillion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + quadrillions + ) + + # WFST grammar for sextillions + sextillion = trilliarde + sextillions = (pynini.cross(trilliarde, "1.000.000.000.000.000.000.000")) | ( + ( + (pynini.cross(sextillion, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(sextillion, ".") + delete_und.ques) + | pynutil.insert("000.") + ) + + delete_space.ques + + quintillions + ) + + # Remove the leading zeros + non_zero_digits = pynini.difference(NEMO_DIGIT, "0") + chars_to_remove = pynini.accep("0") | pynini.accep(".") + remove_chars = pynutil.delete(pynini.closure(chars_to_remove)) + remove_leading_zeros = pynini.cdrewrite(remove_chars, "[BOS]", non_zero_digits, NEMO_SIGMA) + + # All together now + grammars = [ + sextillions, + quintillions, + quadrillions, + trillions, + billions, + millions, + thousands, + digit_cluster, + zero, + ] + + graph_cardinals = "" + for grammar in grammars: + graph_cardinals |= grammar + + # Generates a graph accepting all digits to be passed to other semiotic classes + graph_everything = graph_cardinals @ remove_leading_zeros + self.graph_all_cardinals = graph_everything.optimize() - # add_space_between_chars = pynini.cdrewrite(pynini.closure(insert_space, 0, 1), NEMO_CHAR, NEMO_CHAR, NEMO_SIGMA) - optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ")) + # Generates a graph denormalizing years from 0 to 9999 + # The graph will be passed into other semiotic classes + # Years 0 - 999 denormalize as regular cardinals + first_millenium = non_zero_digit_cluster # | zero + second_tenth_millenium = non_zero_thousands + # The graph below covers exceptions + # e.g. years 1100 - 1999 + # and all colloquial expresions (e.g. zwanzigvierundzwanzig -> 2024) + ten = pynini.project(irregular_teens @ pynini.accep("10"), "input") + remove_ten = pynini.project(graph_10_99, "input") - ten + graph_11_99 = remove_ten @ graph_10_99 - graph = (tn_cardinal_tagger.graph @ optional_delete_space).invert().optimize() - self.graph_hundred_component_at_least_one_none_zero_digit = ( - (tn_cardinal_tagger.graph_hundred_component_at_least_one_none_zero_digit @ optional_delete_space) - .invert() - .optimize() + years_exceptions = ( + graph_11_99 + + pynutil.delete(NEMO_SPACE).ques + + pynutil.delete(hundert).ques + + pynutil.delete(NEMO_SPACE).ques + + (graph_10_99 | pynutil.insert("00")) ) + years = first_millenium | second_tenth_millenium | years_exceptions + remove_period_separators = pynini.cdrewrite(pynutil.delete("."), "", "", NEMO_SIGMA) + years = years @ remove_leading_zeros @ remove_period_separators + self.graph_years = years.optimize() - self.graph_ties = (tn_cardinal_tagger.two_digit_non_zero @ optional_delete_space).invert().optimize() - # this is to make sure if there is an ambiguity with decimal, decimal is chosen, e.g. 1000000 vs. 1 million - graph = pynutil.add_weight(graph, weight=0.001) - self.graph_no_exception = graph - self.digit = pynini.arcmap(tn_cardinal_tagger.digit, map_type="rmweight").invert().optimize() - graph_exception = pynini.project(self.digit, 'input') - self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + # The block below leaves numerals 1 - 12 canonically normalized + accept_denormalized_first_dozen = pynini.project(to_denormalize, "input") # acceptor for null - zwölf + accept_denormalized_everything = pynini.project( + self.graph_all_cardinals, "input" + ) # acceptor for all verbalized cardinals + accept_without_first_dozen = ( + accept_denormalized_everything - accept_denormalized_first_dozen + ) # acceptor for all verbalized cardinals greater than 12 + transduce_without_first_dozen = ( + accept_without_first_dozen @ self.graph_all_cardinals + ) # transducer for all verbalized cardinals greater than 12 + graph = accept_denormalized_first_dozen | transduce_without_first_dozen + self.graph = graph.optimize() - self.optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", "\"-\" "), 0, 1 + self.optional_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross(minus + pynini.accep(" "), '"-"') + pynutil.insert(" "), + 0, + 1, ) - final_graph = self.optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + all_cardinals_graph = ( + self.optional_negative + pynutil.insert('integer: "') + self.graph_all_cardinals + pynutil.insert('"') + ) + self.all_cardinals_graph = all_cardinals_graph.optimize() + + # The final graph for this semiotic class leaves the first dozen normalized + final_graph = self.optional_negative + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') + + # Canonical representation with the first dozen normalized + self.canonical_cardinals_graph = final_graph.optimize() final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() + + self.graph_no_exception = self.graph_all_cardinals + self.optional_minus_graph = self.optional_negative + self.graph_hundred_component_at_least_one_none_zero_digit = self.graph_all_cardinals + self.digit = self.digits + self.graph_ties = self.graph_double_digits diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py index 1d60d071a..e0dbc4a42 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py @@ -93,7 +93,7 @@ def __init__( tn_electronic_verbalizer = TNElectronicVerbalizer(deterministic=False) tn_whitelist_tagger = TNWhitelistTagger(input_case="cased", deterministic=False, input_file=whitelist) - cardinal = CardinalFst(tn_cardinal_tagger=tn_cardinal_tagger) + cardinal = CardinalFst() cardinal_graph = cardinal.fst ordinal = OrdinalFst(itn_cardinal_tagger=cardinal, tn_ordinal_verbalizer=tn_ordinal_verbalizer) diff --git a/nemo_text_processing/inverse_text_normalization/de/utils.py b/nemo_text_processing/inverse_text_normalization/de/utils.py new file mode 100644 index 000000000..78fdd87f2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/utils.py @@ -0,0 +1,27 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index b13382a8e..ba8cfc8b9 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -15,22 +15,81 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path class CardinalFst(GraphFst): """ - Finite state transducer for verbalizing cardinal - e.g. cardinal { integer: "23" negative: "-" } -> -23 - - Args: - tn_cardinal_verbalizer: TN cardinal verbalizer + Finite state transducer for verbalizing cardinal numbers. Note that the verbalizer retains period-separated formatting. + e.g. 'cardinal { negative: "-" integer: "1.234.512.102" }' -> -1.234.512.102 """ - def __init__(self, tn_cardinal_verbalizer: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) - self.numbers = tn_cardinal_verbalizer.numbers - optional_sign = pynini.closure(pynutil.delete("negative: \"") + NEMO_NOT_QUOTE + pynutil.delete("\" "), 0, 1) - graph = optional_sign + self.numbers - delete_tokens = self.delete_tokens(graph) + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + DE_chars = pynini.union(*"äöüÄÖÜß").optimize() + + # removes the 'negative:' label and leaves the optional '-' sign in place + optional_minus = pynini.closure( + pynutil.delete("negative:") + + pynutil.delete(NEMO_SPACE) + + pynutil.delete('"') + + pynini.accep("-") + + pynutil.delete('"') + + pynutil.delete(NEMO_SPACE), + 0, + 1, + ) + + # handles all elements of a cardinal integer + integer_chars = NEMO_DIGIT | pynini.accep(".") + cardinal_components = NEMO_DIGIT | NEMO_ALPHA | DE_chars | pynini.accep(".") + + # removes the 'integer:' label + just_integers = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(integer_chars, 1) + + pynutil.delete('"') + + delete_space + ) + + # handles the canonical representation with the first dozen normalized + first_dozen_verbalized = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(cardinal_components, 1) + + pynutil.delete('"') + + delete_space + ) + + # Handles noun + number combinations, where the noun forces full denormalization + # The nouns are implemented as a .tsv list + nouns_forcing_denormalization = pynini.string_file( + get_abs_path("data/measure/nouns_forcing_denormalization.tsv") + ) + graph_forced_denormalization = ( + pynutil.delete("morphosyntactic_features: ") + + pynutil.delete('"') + + nouns_forcing_denormalization + + pynutil.delete('"') + + pynini.accep(NEMO_SPACE) + + just_integers + ) + + graph = optional_minus + just_integers + self.numbers = graph.optimize() + first_dozen = (optional_minus + first_dozen_verbalized).optimize() + self.first_dozen = first_dozen + updated_cardinals = (first_dozen | graph_forced_denormalization).optimize() + delete_tokens = self.delete_tokens(updated_cardinals) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py index 9c921a63a..7aa670991 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py @@ -17,7 +17,6 @@ from nemo_text_processing.inverse_text_normalization.de.verbalizers.measure import MeasureFst from nemo_text_processing.inverse_text_normalization.de.verbalizers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.de.verbalizers.time import TimeFst -from nemo_text_processing.text_normalization.de.verbalizers.cardinal import CardinalFst as TNCardinalVerbalizer from nemo_text_processing.text_normalization.de.verbalizers.decimal import DecimalFst as TNDecimalVerbalizer from nemo_text_processing.text_normalization.en.graph_utils import GraphFst @@ -31,10 +30,9 @@ class VerbalizeFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) - tn_cardinal_verbalizer = TNCardinalVerbalizer(deterministic=False) tn_decimal_verbalizer = TNDecimalVerbalizer(deterministic=False) - cardinal = CardinalFst(tn_cardinal_verbalizer=tn_cardinal_verbalizer) + cardinal = CardinalFst() cardinal_graph = cardinal.fst decimal = DecimalFst(tn_decimal_verbalizer=tn_decimal_verbalizer) decimal_graph = decimal.fst diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt index 0b2064296..6572163db 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt @@ -4,26 +4,26 @@ ein hundert und zwei~102 einhundertzwei~102 ein hundert und zwanzig~120 ein hundert und elf~111 -ein tausend~1000 -eintausend~1000 +ein tausend~1.000 +eintausend~1.000 ein hundert zwanzig~120 -ein tausend zwanzig~1020 -eintausendzwanzig~1020 -neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9789382536130 +ein tausend zwanzig~1.020 +eintausendzwanzig~1.020 +neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9.789.382.536.130 zwei hundert vier und fünfzig~254 -ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147451 -eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1156173 -eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1593072961 -sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97808264772792005 -zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10010000010100010 -zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10010000010100010 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünfundzwanzigtausendsiebenunddreißig~-25037 -eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1264301938000104 -eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1264301938000104 +ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147.451 +eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1.156.173 +eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1.593.072.961 +sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97.808.264.772.792.005 +zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10.010.000.010.100.010 +zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10.010.000.010.100.010 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünfundzwanzigtausendsiebenunddreißig~-25.037 +eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1.264.301.938.000.104 +eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1.264.301.938.000.104 minus sechzig~-60 -sechs und vierzig tausend sechs hundert vier und sechzig~46664 +sechsundvierzig tausend sechshundert vierundsechzig~46.664 sechzig~60 null~null eins~eins @@ -31,10 +31,12 @@ ein~ein eine~eine einer~einer zwei~zwei +zwö~zwö +zwo~zwo neun~neun -zehn~10 -elf~11 -zwölf~12 +zehn~zehn +elf~elf +zwölf~zwölf dreizehn~13 vierzehn~14 fünfzehn~15 @@ -49,14 +51,18 @@ sechzig~60 siebzig~70 achtzig~80 neunzig~90 -zwei millionen drei~2000003 -ein tausend dreizehn~1013 -ein tausend eins~1001 -ein tausend ein hundert~1100 -ein tausend sechs und zwanzig~1026 -ein tausend ein hundert sechs und zwanzig~1126 -achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18450990 -achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18940722 -achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18690916 -achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18690916 -achtzehn tausend acht hundert achtzig~18880 +zwei millionen drei~2.000.003 +ein tausend dreizehn~1.013 +ein tausend eins~1.001 +ein tausend ein hundert~1.100 +ein tausend sechs und zwanzig~1.026 +ein tausend ein hundert sechs und zwanzig~1.126 +achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18.450.990 +achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18.940.722 +achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18.690.916 +achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18.690.916 +achtzehn tausend acht hundert achtzig~18.880 +einhunderteins~101 +ein tausend einhundert sechsundzwanzig~1.126 +Nummer acht~Nummer 8 +Abteil sechs~Abteil 6 \ No newline at end of file