From 66ee78ee07beb10d1b18e7248c02a1640cefb955 Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Wed, 2 Sep 2026 12:02:28 +0000 Subject: [PATCH 1/2] percent and time bug fixes Signed-off-by: Shreyas Pawar --- Jenkinsfile | 2 +- .../hi/data/measure/unit.tsv | 3 +- .../text_normalization/hi/taggers/measure.py | 42 +++++++++++++++++-- .../text_normalization/hi/taggers/time.py | 9 ++-- .../test_cases_measure.txt | 4 +- .../test_cases_time.txt | 4 +- 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..a2de3c8c4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-02-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv index d236dd51b..6598affc0 100644 --- a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv +++ b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv @@ -142,4 +142,5 @@ mi/hr मील प्रति घंटा mi/min मील प्रति मिनट ₹/ac रुपए प्रति एकड़ x बाई -* बाई \ No newline at end of file +* बाई +% प्रतिशत \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 67043b727..cf8068b42 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -228,9 +228,22 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser year_informal = pynini.string_map([("yr", "साल")]) year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv")) - # All units EXCEPT year - unit_inputs_except_yr = pynini.difference(pynini.project(unit_graph, "input"), pynini.accep("yr")) - unit_graph_no_year = pynini.compose(unit_inputs_except_yr, unit_graph) + # All units EXCEPT year and percent + unit_inputs_regular = pynini.difference( + pynini.project(unit_graph, "input"), + pynini.union(pynini.accep("yr"), pynini.accep("%")) + ) + unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph) + + percent_graph = pynini.compose(pynini.accep("%"), unit_graph) + + percent_unit = ( + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + percent_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) # Load quarterly units from separate files: map (FST) and list (FSA) quarterly_units_map = pynini.string_file(get_abs_path("data/measure/quarterly_units_map.tsv")) @@ -454,14 +467,37 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") ) + graph_cardinal_percent = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + pynini.closure(delete_space, 0, 1) + + percent_unit + ) + + graph_decimal_percent = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal_graph + + pynutil.insert(" }") + + pynini.closure(delete_space, 0, 1) + + percent_unit + ) + address_graph = self.get_address_graph(cardinal, ordinal, serial, input_case) structured_address_graph = self.get_structured_address_graph(cardinal, ordinal, input_case) graph = ( pynutil.add_weight(graph_decimal, 0.1) | pynutil.add_weight(graph_decimal_year_formal, 0.1) + | pynutil.add_weight(graph_decimal_percent, 0.1) | pynutil.add_weight(graph_cardinal, 0.1) | pynutil.add_weight(graph_cardinal_year_formal, 0.1) + | pynutil.add_weight(graph_cardinal_percent, 0.1) | pynutil.add_weight(graph_cardinal_year_informal, -0.1) # Higher priority for small numbers | pynutil.add_weight(graph_exceptions, 0.1) | pynutil.add_weight(graph_dedh_dhai, -0.2) diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py index fc598c2b1..68568aa60 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/time.py +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -55,9 +55,10 @@ def __init__(self, cardinal: GraphFst): super().__init__(name="time", kind="classify") delete_colon = pynutil.delete(":") - cardinal_graph = cardinal.digit | cardinal.teens_and_ties + delete_leading_zero = pynini.closure(pynutil.delete("0") | pynutil.delete("०"), 0, 1) + cardinal_graph = delete_leading_zero + (cardinal.digit | cardinal.teens_and_ties) - self.hours = pynutil.insert("hours: \"") + hours_graph + pynutil.insert("\" ") + self.hours = pynutil.insert("hours: \"") + delete_leading_zero + hours_graph + pynutil.insert("\" ") self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ") self.seconds = pynutil.insert("seconds: \"") + seconds_graph + pynutil.insert("\" ") @@ -73,7 +74,7 @@ def __init__(self, cardinal: GraphFst): graph_h = self.hours + delete_colon + pynutil.delete(HI_DOUBLE_ZERO) # Support all combinations of Devanagari and Arabic digits for dedh/dhai patterns - dedh_dhai_graph = pynini.string_map( + dedh_dhai_graph = delete_leading_zero + pynini.string_map( [ ("१:३०", HI_DEDH), ("१:30", HI_DEDH), @@ -92,7 +93,7 @@ def __init__(self, cardinal: GraphFst): sadhe_numbers = cardinal_graph + pynini.cross(HI_TIME_THIRTY, "") sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers - paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) + paune = delete_leading_zero + pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) paune_numbers = paune + pynini.cross(HI_TIME_FORTYFIVE, "") paune_graph = pynutil.insert(HI_PAUNE) + pynutil.insert(NEMO_SPACE) + paune_numbers diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt index 6afd66b7f..13f0cda5e 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt @@ -66,4 +66,6 @@ ५ yr~पाँच साल 1.5 yr~डेढ़ साल २.५ yr~ढाई साल -3.5 yr~साढ़े तीन साल \ No newline at end of file +3.5 yr~साढ़े तीन साल +100%~एक सौ प्रतिशत +५०.५० %~पचास दशमलव पाँच शून्य प्रतिशत \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt index 5bc796209..2e613ddae 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt @@ -15,4 +15,6 @@ दोपहर के 3:00~दोपहर के तीन बजे रात के १०:४८:५०~रात के दस बजकर अड़तालीस मिनट पचास सेकंड रात के 11:50~रात के ग्यारह बजकर पचास मिनट -रात के ८:००~रात के आठ बजे \ No newline at end of file +रात के ८:००~रात के आठ बजे +आज सुबह ०९:१५ बजे~आज सुबह सवा नौ बजे +08:00~आठ बजे \ No newline at end of file From 4e60c63fb34cfbab727c6121f5521bcf2e84eea9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:28:40 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/hi/taggers/measure.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index cf8068b42..897f77fac 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -230,13 +230,12 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser # All units EXCEPT year and percent unit_inputs_regular = pynini.difference( - pynini.project(unit_graph, "input"), - pynini.union(pynini.accep("yr"), pynini.accep("%")) + pynini.project(unit_graph, "input"), pynini.union(pynini.accep("yr"), pynini.accep("%")) ) unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph) percent_graph = pynini.compose(pynini.accep("%"), unit_graph) - + percent_unit = ( pynutil.insert(NEMO_SPACE) + pynutil.insert("units: \"") @@ -475,7 +474,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") + pynutil.insert(NEMO_SPACE) + pynutil.insert("}") - + pynini.closure(delete_space, 0, 1) + + pynini.closure(delete_space, 0, 1) + percent_unit ) @@ -484,7 +483,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + optional_graph_negative + decimal_graph + pynutil.insert(" }") - + pynini.closure(delete_space, 0, 1) + + pynini.closure(delete_space, 0, 1) + percent_unit )