diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..ce326e63f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/money/decimal.tsv b/nemo_text_processing/text_normalization/hi/data/money/decimal.tsv new file mode 100644 index 000000000..2a7601f76 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/money/decimal.tsv @@ -0,0 +1 @@ +. दशमलव \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/money/quantities.tsv b/nemo_text_processing/text_normalization/hi/data/money/quantities.tsv new file mode 100644 index 000000000..66d556c36 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/money/quantities.tsv @@ -0,0 +1,10 @@ +हजार +हज़ार +लाख +करोड़ +करोड +अरब +खरब +मिलियन +बिलियन +ट्रिलियन \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/money.py b/nemo_text_processing/text_normalization/hi/taggers/money.py index 16f389ae7..010b94f0d 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/money.py +++ b/nemo_text_processing/text_normalization/hi/taggers/money.py @@ -45,6 +45,9 @@ def __init__(self, cardinal: GraphFst): _deva_to_ascii = pynini.invert(_en_to_hi_digit) deva_to_ascii = pynini.closure(_deva_to_ascii | pynini.union(*"0123456789"), 1) + decimal_tsv = pynini.string_file(get_abs_path("data/money/decimal.tsv")) + decimal_separator = pynini.project(decimal_tsv, "input") + _ascii_digit = pynini.union(*"0123456789") _ascii_nonzero = pynini.union(*"123456789") _deva_nonzero = pynini.union(*"१२३४५६७८९") @@ -77,7 +80,7 @@ def __init__(self, cardinal: GraphFst): ) optional_delete_fractional_zeros = pynini.closure( - pynutil.delete(".") + pynini.closure(pynutil.delete("0") | pynutil.delete("०"), 1), + pynutil.delete(decimal_separator) + pynini.closure(pynutil.delete("0") | pynutil.delete("०"), 1), 0, 1, ) @@ -92,6 +95,40 @@ def __init__(self, cardinal: GraphFst): ) guarded_decimal_digits = has_3plus_sig_digits @ decimal_digits + # Load scale words dynamically from TSV + quantity_graph = ( + pynutil.insert('quantity: "') + + pynini.string_file(get_abs_path("data/money/quantities.tsv")).optimize() + + pynutil.insert('"') + ) + + # Allow an optional space between the number and the quantity word + delete_space_opt = pynini.closure(pynutil.delete(" "), 0, 1) + + # Path for scaled money with decimals (e.g. ₹२.५ करोड़) + graph_scaled_decimal = ( + optional_graph_negative + + currency_major + + insert_space + + integer + + pynini.cross(decimal_separator, " ") + + decimal_digits + + delete_space_opt + + insert_space + + quantity_graph + ).optimize() + + # Path for scaled money without decimals (e.g. ₹५ करोड़) + graph_scaled_major = ( + optional_graph_negative + + currency_major + + insert_space + + integer + + delete_space_opt + + insert_space + + quantity_graph + ).optimize() + graph_decimal_path = ( optional_graph_negative + currency_major @@ -99,7 +136,7 @@ def __init__(self, cardinal: GraphFst): + pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') - + pynini.cross(".", " ") + + pynini.cross(decimal_separator, " ") + guarded_decimal_digits ).optimize() @@ -134,7 +171,7 @@ def _build_major_and_minor(sym_maj_labels, int_graph): + curr_maj + insert_space + int_graph - + pynini.cross(".", " ") + + pynini.cross(decimal_separator, " ") + fraction + insert_space + curr_min @@ -150,6 +187,8 @@ def _build_major_and_minor(sym_maj_labels, int_graph): graph_currencies = ( pynutil.add_weight(graph_major_only_singular | graph_major_and_minor_singular, -0.001) | pynutil.add_weight(graph_decimal_path, -0.0005) + | pynutil.add_weight(graph_scaled_decimal, -0.0002) + | pynutil.add_weight(graph_scaled_major, -0.0002) | graph_major_only | graph_major_and_minor ) diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/money.py b/nemo_text_processing/text_normalization/hi/verbalizers/money.py index 1e5da99e4..d9f796587 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/money.py @@ -44,8 +44,17 @@ def __init__(self): pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') ) + decimal_tsv = pynini.string_file(get_abs_path("data/money/decimal.tsv")) + decimal_word = pynini.project(decimal_tsv, "output") + currency_minor = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + quantity = pynutil.delete('quantity: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + drop_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + sp + pynutil.delete("true") + sp, 0, 1 + ) + graph_major_only = integer_part + sp + currency_major all_major_names = [maj for maj, _ in load_labels(get_abs_path("data/money/major_minor_currencies.tsv"))] @@ -90,13 +99,58 @@ def __init__(self): + sp + integer_part + sp - + pynutil.insert(" दशमलव ") + + pynutil.insert(" ") + + pynutil.insert(decimal_word) + + pynutil.insert(" ") + fractional_part + pynutil.insert(NEMO_SPACE) + pynutil.insert(major) ) graph_decimal_money = pynini.union(*decimal_graphs) - graph = graph_major_only | graph_major_minor | pynutil.add_weight(graph_minor_only, -0.1) | graph_decimal_money + scaled_decimal_graphs = [] + scaled_major_graphs = [] + + for major in all_major_names: + graph_major_slot = pynutil.delete('currency_maj: "') + pynutil.delete(major) + pynutil.delete('"') + + scaled_decimal_graphs.append( + drop_preserve_order + + graph_major_slot + + sp + + integer_part + + sp + + pynutil.insert(" ") + + pynutil.insert(decimal_word) + + pynutil.insert(" ") + + fractional_part + + sp + + quantity + + pynutil.insert(NEMO_SPACE) + + pynutil.insert(major) + ) + + scaled_major_graphs.append( + drop_preserve_order + + graph_major_slot + + sp + + integer_part + + sp + + quantity + + pynutil.insert(NEMO_SPACE) + + pynutil.insert(major) + ) + + graph_scaled_decimal = pynini.union(*scaled_decimal_graphs) + graph_scaled_major = pynini.union(*scaled_major_graphs) + + graph = ( + graph_major_only + | graph_major_minor + | pynutil.add_weight(graph_minor_only, -0.1) + | graph_decimal_money + | graph_scaled_decimal + | graph_scaled_major + ) self.fst = self.delete_tokens(graph).optimize() diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt index e5f157872..ea89ac6e5 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt @@ -141,4 +141,9 @@ $1.2000~एक डॉलर बीस सेंट ₹५,५६,३२०~पाँच लाख छप्पन हज़ार तीन सौ बीस रुपए ₹१,००,०००~एक लाख रुपए ₹२,१४८~दो हज़ार एक सौ अड़तालीस रुपए -₹९९,९९९~निन्यानबे हज़ार नौ सौ निन्यानबे रुपए \ No newline at end of file +₹९९,९९९~निन्यानबे हज़ार नौ सौ निन्यानबे रुपए +₹२.५ करोड़~दो दशमलव पाँच करोड़ रुपए +₹५ करोड़~पाँच करोड़ रुपए +$१.५ मिलियन~एक दशमलव पाँच मिलियन डॉलर +£१२ लाख~बारह लाख पाउंड +₹१.५ हजार~एक दशमलव पाँच हजार रुपए