diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..a2de3c8c4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-02-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv index d236dd51b..6598affc0 100644 --- a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv +++ b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv @@ -142,4 +142,5 @@ mi/hr मील प्रति घंटा mi/min मील प्रति मिनट ₹/ac रुपए प्रति एकड़ x बाई -* बाई \ No newline at end of file +* बाई +% प्रतिशत \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 67043b727..897f77fac 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -228,9 +228,21 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser year_informal = pynini.string_map([("yr", "साल")]) year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv")) - # All units EXCEPT year - unit_inputs_except_yr = pynini.difference(pynini.project(unit_graph, "input"), pynini.accep("yr")) - unit_graph_no_year = pynini.compose(unit_inputs_except_yr, unit_graph) + # All units EXCEPT year and percent + unit_inputs_regular = pynini.difference( + pynini.project(unit_graph, "input"), pynini.union(pynini.accep("yr"), pynini.accep("%")) + ) + unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph) + + percent_graph = pynini.compose(pynini.accep("%"), unit_graph) + + percent_unit = ( + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + percent_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) # Load quarterly units from separate files: map (FST) and list (FSA) quarterly_units_map = pynini.string_file(get_abs_path("data/measure/quarterly_units_map.tsv")) @@ -454,14 +466,37 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") ) + graph_cardinal_percent = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + pynini.closure(delete_space, 0, 1) + + percent_unit + ) + + graph_decimal_percent = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal_graph + + pynutil.insert(" }") + + pynini.closure(delete_space, 0, 1) + + percent_unit + ) + address_graph = self.get_address_graph(cardinal, ordinal, serial, input_case) structured_address_graph = self.get_structured_address_graph(cardinal, ordinal, input_case) graph = ( pynutil.add_weight(graph_decimal, 0.1) | pynutil.add_weight(graph_decimal_year_formal, 0.1) + | pynutil.add_weight(graph_decimal_percent, 0.1) | pynutil.add_weight(graph_cardinal, 0.1) | pynutil.add_weight(graph_cardinal_year_formal, 0.1) + | pynutil.add_weight(graph_cardinal_percent, 0.1) | pynutil.add_weight(graph_cardinal_year_informal, -0.1) # Higher priority for small numbers | pynutil.add_weight(graph_exceptions, 0.1) | pynutil.add_weight(graph_dedh_dhai, -0.2) diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py index fc598c2b1..68568aa60 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/time.py +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -55,9 +55,10 @@ def __init__(self, cardinal: GraphFst): super().__init__(name="time", kind="classify") delete_colon = pynutil.delete(":") - cardinal_graph = cardinal.digit | cardinal.teens_and_ties + delete_leading_zero = pynini.closure(pynutil.delete("0") | pynutil.delete("०"), 0, 1) + cardinal_graph = delete_leading_zero + (cardinal.digit | cardinal.teens_and_ties) - self.hours = pynutil.insert("hours: \"") + hours_graph + pynutil.insert("\" ") + self.hours = pynutil.insert("hours: \"") + delete_leading_zero + hours_graph + pynutil.insert("\" ") self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ") self.seconds = pynutil.insert("seconds: \"") + seconds_graph + pynutil.insert("\" ") @@ -73,7 +74,7 @@ def __init__(self, cardinal: GraphFst): graph_h = self.hours + delete_colon + pynutil.delete(HI_DOUBLE_ZERO) # Support all combinations of Devanagari and Arabic digits for dedh/dhai patterns - dedh_dhai_graph = pynini.string_map( + dedh_dhai_graph = delete_leading_zero + pynini.string_map( [ ("१:३०", HI_DEDH), ("१:30", HI_DEDH), @@ -92,7 +93,7 @@ def __init__(self, cardinal: GraphFst): sadhe_numbers = cardinal_graph + pynini.cross(HI_TIME_THIRTY, "") sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers - paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) + paune = delete_leading_zero + pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) paune_numbers = paune + pynini.cross(HI_TIME_FORTYFIVE, "") paune_graph = pynutil.insert(HI_PAUNE) + pynutil.insert(NEMO_SPACE) + paune_numbers diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt index 6afd66b7f..13f0cda5e 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt @@ -66,4 +66,6 @@ ५ yr~पाँच साल 1.5 yr~डेढ़ साल २.५ yr~ढाई साल -3.5 yr~साढ़े तीन साल \ No newline at end of file +3.5 yr~साढ़े तीन साल +100%~एक सौ प्रतिशत +५०.५० %~पचास दशमलव पाँच शून्य प्रतिशत \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt index 5bc796209..2e613ddae 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt @@ -15,4 +15,6 @@ दोपहर के 3:00~दोपहर के तीन बजे रात के १०:४८:५०~रात के दस बजकर अड़तालीस मिनट पचास सेकंड रात के 11:50~रात के ग्यारह बजकर पचास मिनट -रात के ८:००~रात के आठ बजे \ No newline at end of file +रात के ८:००~रात के आठ बजे +आज सुबह ०९:१५ बजे~आज सुबह सवा नौ बजे +08:00~आठ बजे \ No newline at end of file