Hi TN: percent (measure class) and time bug fixes

Jenkinsfile

@@ -29,7 +29,7 @@ pipeline {
     MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
     KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6'
-    KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1'
+    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-02-26-0'
     DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/text_normalization/hi/data/measure/unit.tsv

@@ -142,4 +142,5 @@ mi/hr	मील प्रति घंटा
 mi/min	मील प्रति मिनट
 ₹/ac	रुपए प्रति एकड़
 x	बाई
-*	बाई
+*	बाई
+%	प्रतिशत

nemo_text_processing/text_normalization/hi/taggers/measure.py

@@ -228,9 +228,21 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser
         year_informal = pynini.string_map([("yr", "साल")])
         year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv"))
 
-        # All units EXCEPT year
-        unit_inputs_except_yr = pynini.difference(pynini.project(unit_graph, "input"), pynini.accep("yr"))
-        unit_graph_no_year = pynini.compose(unit_inputs_except_yr, unit_graph)
+        # All units EXCEPT year and percent
+        unit_inputs_regular = pynini.difference(
+            pynini.project(unit_graph, "input"), pynini.union(pynini.accep("yr"), pynini.accep("%"))
+        )
+        unit_graph_no_year = pynini.compose(unit_inputs_regular, unit_graph)
+
+        percent_graph = pynini.compose(pynini.accep("%"), unit_graph)
+
+        percent_unit = (
+            pynutil.insert(NEMO_SPACE)
+            + pynutil.insert("units: \"")
+            + percent_graph
+            + pynutil.insert("\"")
+            + pynutil.insert(NEMO_SPACE)
+        )
 
         # Load quarterly units from separate files: map (FST) and list (FSA)
         quarterly_units_map = pynini.string_file(get_abs_path("data/measure/quarterly_units_map.tsv"))
@@ -454,14 +466,37 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser
             + pynutil.insert("\"")
         )
 
+        graph_cardinal_percent = (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert("integer: \"")
+            + cardinal_graph
+            + pynutil.insert("\"")
+            + pynutil.insert(NEMO_SPACE)
+            + pynutil.insert("}")
+            + pynini.closure(delete_space, 0, 1)
+            + percent_unit
+        )
+
+        graph_decimal_percent = (
+            pynutil.insert("decimal { ")
+            + optional_graph_negative
+            + decimal_graph
+            + pynutil.insert(" }")
+            + pynini.closure(delete_space, 0, 1)
+            + percent_unit
+        )
+
         address_graph = self.get_address_graph(cardinal, ordinal, serial, input_case)
         structured_address_graph = self.get_structured_address_graph(cardinal, ordinal, input_case)
 
         graph = (
             pynutil.add_weight(graph_decimal, 0.1)
             | pynutil.add_weight(graph_decimal_year_formal, 0.1)
+            | pynutil.add_weight(graph_decimal_percent, 0.1)
             | pynutil.add_weight(graph_cardinal, 0.1)
             | pynutil.add_weight(graph_cardinal_year_formal, 0.1)
+            | pynutil.add_weight(graph_cardinal_percent, 0.1)
             | pynutil.add_weight(graph_cardinal_year_informal, -0.1)  # Higher priority for small numbers
             | pynutil.add_weight(graph_exceptions, 0.1)
             | pynutil.add_weight(graph_dedh_dhai, -0.2)

nemo_text_processing/text_normalization/hi/taggers/time.py

@@ -55,9 +55,10 @@ def __init__(self, cardinal: GraphFst):
         super().__init__(name="time", kind="classify")
 
         delete_colon = pynutil.delete(":")
-        cardinal_graph = cardinal.digit | cardinal.teens_and_ties
+        delete_leading_zero = pynini.closure(pynutil.delete("0") | pynutil.delete("०"), 0, 1)
+        cardinal_graph = delete_leading_zero + (cardinal.digit | cardinal.teens_and_ties)
 
-        self.hours = pynutil.insert("hours: \"") + hours_graph + pynutil.insert("\" ")
+        self.hours = pynutil.insert("hours: \"") + delete_leading_zero + hours_graph + pynutil.insert("\" ")
         self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ")
         self.seconds = pynutil.insert("seconds: \"") + seconds_graph + pynutil.insert("\" ")
 
@@ -73,7 +74,7 @@ def __init__(self, cardinal: GraphFst):
         graph_h = self.hours + delete_colon + pynutil.delete(HI_DOUBLE_ZERO)
 
         # Support all combinations of Devanagari and Arabic digits for dedh/dhai patterns
-        dedh_dhai_graph = pynini.string_map(
+        dedh_dhai_graph = delete_leading_zero + pynini.string_map(
             [
                 ("१:३०", HI_DEDH),
                 ("१:30", HI_DEDH),
@@ -92,7 +93,7 @@ def __init__(self, cardinal: GraphFst):
         sadhe_numbers = cardinal_graph + pynini.cross(HI_TIME_THIRTY, "")
         sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers
 
-        paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv"))
+        paune = delete_leading_zero + pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv"))
         paune_numbers = paune + pynini.cross(HI_TIME_FORTYFIVE, "")
         paune_graph = pynutil.insert(HI_PAUNE) + pynutil.insert(NEMO_SPACE) + paune_numbers
 

tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt

@@ -66,4 +66,6 @@
 ५ yr~पाँच साल
 1.5 yr~डेढ़ साल
 २.५ yr~ढाई साल
-3.5 yr~साढ़े तीन साल
+3.5 yr~साढ़े तीन साल
+100%~एक सौ प्रतिशत
+५०.५० %~पचास दशमलव पाँच शून्य प्रतिशत

tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt

@@ -15,4 +15,6 @@
 दोपहर के 3:00~दोपहर के तीन बजे
 रात के १०:४८:५०~रात के दस बजकर अड़तालीस मिनट पचास सेकंड 
 रात के 11:50~रात के ग्यारह बजकर पचास मिनट 
-रात के ८:००~रात के आठ बजे
+रात के ८:००~रात के आठ बजे
+आज सुबह ०९:१५ बजे~आज सुबह सवा नौ बजे
+08:00~आठ बजे