Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
856a301
feat(ta): add cardinal TN tagger, verbalizer and test cases
surendran-246 Jun 4, 2026
7ebabdf
feat(ta): address review comments
surendran-246 Jun 5, 2026
d68701b
message
surendran-246 Jun 16, 2026
c2fc100
Updated Ta Cardinal
surendran-246 Jun 17, 2026
f9d575d
cardinal implementation
surendran-246 Jul 7, 2026
fcfcb3e
cardinal file
surendran-246 Jul 7, 2026
423f239
Updated cardinal file
surendran-246 Jul 7, 2026
5a39ceb
Updated cardinal
surendran-246 Jul 7, 2026
a0815e9
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 7, 2026
eaf2223
Cardinal updated file
surendran-246 Jul 7, 2026
c0e51f1
Updated cardinal
surendran-246 Jul 7, 2026
a0cd909
Updated Cardinal
surendran-246 Jul 8, 2026
4761e67
Added Cardinal
surendran-246 Jul 8, 2026
301db6f
Ta cardinal
surendran-246 Jul 8, 2026
60b9d58
Added Cardinal file
surendran-246 Jul 8, 2026
fe805eb
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 8, 2026
e2e7996
Updated Cardinal file
surendran-246 Jul 8, 2026
2907744
Updated TA Cardinal
surendran-246 Jul 23, 2026
0531bc2
Merge branch 'staging/tamil_tn_v1' into feat-ta-cardinal
surendran-246 Jul 23, 2026
026875c
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 23, 2026
5561410
Modified Cardinal
surendran-246 Jul 24, 2026
9bbb059
Update Cardinal class
surendran-246 Aug 10, 2026
3982a3d
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Aug 10, 2026
bf50955
Updated Cardinal class based on PR reviews
surendran-246 Aug 18, 2026
b4c1aeb
Updated Cardinal class based on PR reviews
surendran-246 Aug 18, 2026
e63ae9c
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Aug 18, 2026
388ae35
Added docstring for taggers/ word file
surendran-246 Aug 19, 2026
0f657e6
Added docstring file in taggers
surendran-246 Aug 19, 2026
316e82e
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Aug 19, 2026
f64f094
Updated Jenkins file
surendran-246 Aug 26, 2026
7fd892b
Updated Cardinal class based on the PR review
surendran-246 Sep 3, 2026
13368b6
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Sep 3, 2026
57ba53c
Added comma parsing + docstring
surendran-246 Sep 3, 2026
aad3c7a
Added comma parsing + docstring
surendran-246 Sep 3, 2026
316230f
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Sep 3, 2026
ddd96c2
Optimized cardinal file and added the init file
surendran-246 Sep 4, 2026
a7df5e9
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Sep 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ pipeline {
JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5'
KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6'
TA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-21-26-0'
DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
}
stages {
Expand Down Expand Up @@ -113,6 +114,11 @@ pipeline {
}
}

stage('L0: TA TN grammars') {
steps {
sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=ta --text="௧" --cache_dir ${TA_TN_CACHE}'
}
}
stage('L0: Create DE/ES TN/ITN Grammars') {
when {
anyOf {
Expand Down Expand Up @@ -408,6 +414,11 @@ pipeline {
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hi/ -m "not pleasefixme" --cpu --tn_cache_dir ${HI_TN_CACHE}'
}
}
stage('L1: Run all TA TN tests (restore grammars from cache)') {
steps {
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ta/ -m "not pleasefixme" --cpu --tn_cache_dir ${TA_TN_CACHE}'
}
}
stage('L1: Run all Codeswitched ES/EN TN/ITN tests (restore grammars from cache)') {
steps {
sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/es_en/ -m "not pleasefixme" --cpu --tn_cache_dir ${ES_EN_TN_CACHE}'
Expand Down
7 changes: 6 additions & 1 deletion nemo_text_processing/text_normalization/normalize.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ def __init__(

if post_process:
self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache)

elif lang == 'it':
from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst
from nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst
Expand All @@ -191,6 +192,10 @@ def __init__(
elif lang == 'ko':
from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst
from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst
elif lang == 'ta':
from nemo_text_processing.text_normalization.ta.taggers.tokenize_and_classify import ClassifyFst
from nemo_text_processing.text_normalization.ta.verbalizers.verbalize_final import VerbalizeFinalFst

else:
raise NotImplementedError(f"Language {lang} has not been supported yet.")

Expand Down Expand Up @@ -737,7 +742,7 @@ def parse_args():
parser.add_argument(
"--language",
help="language",
choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"],
choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ta", "ko", "vi", "pt"],
default="en",
type=str,
)
Expand Down
2 changes: 1 addition & 1 deletion nemo_text_processing/text_normalization/run_evaluate.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def parse_args():
parser.add_argument(
"--lang",
help="language",
choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt'],
choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ta', 'ko', 'vi', 'pt'],
default="en",
type=str,
)
Expand Down
13 changes: 13 additions & 0 deletions nemo_text_processing/text_normalization/ta/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
13 changes: 13 additions & 0 deletions nemo_text_processing/text_normalization/ta/data/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
18 changes: 18 additions & 0 deletions nemo_text_processing/text_normalization/ta/data/numbers/digit.tsv
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
௧ ஒன்று
௨ இரண்டு
௩ மூன்று
௪ நான்கு
௫ ஐந்து
௬ ஆறு
௭ ஏழு
௮ எட்டு
௯ ஒன்பது
1 ஒன்று
2 இரண்டு
3 மூன்று
4 நான்கு
5 ஐந்து
6 ஆறு
7 ஏழு
8 எட்டு
9 ஒன்பது
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
1 நூ
2 இருநூ
3 முந்நூ
4 நானூ
5 ஐநூ
6 அறுநூ
7 எழுநூ
8 எட்டுநூ
9 ஒன்பதுநூ
௧ நூ
௨ இருநூ
௩ முந்நூ
௪ நானூ
௫ ஐநூ
௬ அறுநூ
௭ எழுநூ
௮ எட்டுநூ
௯ ஒன்பதுநூ
10 changes: 10 additions & 0 deletions nemo_text_processing/text_normalization/ta/data/numbers/scale.tsv
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
hundred_suf_e று
hundred_suf_p ற்று
thousand_suf_e ம்
thousand_suf_p த்து
thousand_word_e ஆயிரம்
thousand_word_p ஆயிரத்து
lakh_word_e லட்சம்
lakh_word_p லட்சத்து
crore_word_e கோடி
crore_word_p கோடியே
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
1 ஒரு
௧ ஒரு
2 இரண்டு
௨ இரண்டு

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there any solution that not listing 10 - 99? (eg 20 - 29 share same rule ಮೂ + digit?)

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In Tamil, combining tens + digit isn't simple concatenation — the words fuse at the joint (sandhi). E.g. "20"+"1" isn't "இருபதுஒன்று", it's இருபத்தொன்று; "20"+"2" fuses to இருபத்திரண்டு. This fusion is specific to digits 1 and 2 and can't be generalized safely, so we keep 10–99 fully listed rather than generating them at runtime.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are the tens + digits greater than 3 concatenation, or also unique formulas that can't generalize?

@surendran-246 surendran-246 Sep 3, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Digits 3–9 do generalize. They follow one rule: tens-stem + digit-word (e.g. இருபதி + மூன்று → இருபத்திமூன்று), generated by filtering digit.tsv down to keys 3–9 excluding 1 and 2, whose keys are named via special_units.tsv instead of being listed individually. Digits 1 and 2 remain listed, as they require an additional vowel-drop during fusion (ஒன்று → தொன்று, இரண்டு → திரண்டு) outside the standard pattern. Teens (11–19) also stay listed, following a separate suppletive formation.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sg! can you push those changes here? I still see the full file

Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
10 பத்து
11 பதினொன்று
12 பன்னிரண்டு
13 பதிமூன்று
14 பதினான்கு
15 பதினைந்து
16 பதினாறு
17 பதினேழு
18 பதினெட்டு
19 பத்தொன்பது
20 இருபது
21 இருபத்தொன்று
22 இருபத்திரண்டு
30 முப்பது
31 முப்பத்தொன்று
32 முப்பத்திரண்டு
40 நாற்பது
41 நாற்பத்தொன்று
42 நாற்பத்திரண்டு
50 ஐம்பது
51 ஐம்பத்தொன்று
52 ஐம்பத்திரண்டு
60 அறுபது
61 அறுபத்தொன்று
62 அறுபத்திரண்டு
70 எழுபது
71 எழுபத்தொன்று
72 எழுபத்திரண்டு
80 எண்பது
81 எண்பத்தொன்று
82 எண்பத்திரண்டு
90 தொண்ணூறு
91 தொண்ணூற்றொன்று
92 தொண்ணூற்றிரண்டு
௧௦ பத்து
௧௧ பதினொன்று
௧௨ பன்னிரண்டு
௧௩ பதிமூன்று
௧௪ பதினான்கு
௧௫ பதினைந்து
௧௬ பதினாறு
௧௭ பதினேழு
௧௮ பதினெட்டு
௧௯ பத்தொன்பது
௨௦ இருபது
௨௧ இருபத்தொன்று
௨௨ இருபத்திரண்டு
௩௦ முப்பது
௩௧ முப்பத்தொன்று
௩௨ முப்பத்திரண்டு
௪௦ நாற்பது
௪௧ நாற்பத்தொன்று
௪௨ நாற்பத்திரண்டு
௫௦ ஐம்பது
௫௧ ஐம்பத்தொன்று
௫௨ ஐம்பத்திரண்டு
௬௦ அறுபது
௬௧ அறுபத்தொன்று
௬௨ அறுபத்திரண்டு
௭௦ எழுபது
௭௧ எழுபத்தொன்று
௭௨ எழுபத்திரண்டு
௮௦ எண்பது
௮௧ எண்பத்தொன்று
௮௨ எண்பத்திரண்டு
௯௦ தொண்ணூறு
௯௧ தொண்ணூற்றொன்று
௯௨ தொண்ணூற்றிரண்டு
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
2 இருபத்தி
3 முப்பத்தி
4 நாற்பத்தி
5 ஐம்பத்தி
6 அறுபத்தி
7 எழுபத்தி
8 எண்பத்தி
9 தொண்ணூற்றி
௨ இருபத்தி
௩ முப்பத்தி
௪ நாற்பத்தி
௫ ஐம்பத்தி
௬ அறுபத்தி
௭ எழுபத்தி
௮ எண்பத்தி
௯ தொண்ணூற்றி
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
1 ஆயிர
2 இரண்டாயிர
3 மூவாயிர
4 நான்காயிர
5 ஐந்தாயிர
6 ஆறாயிர
7 ஏழாயிர
8 எட்டாயிர
9 ஒன்பதாயிர
௧ ஆயிர
௨ இரண்டாயிர
௩ மூவாயிர
௪ நான்காயிர
௫ ஐந்தாயிர
௬ ஆறாயிர
௭ ஏழாயிர
௮ எட்டாயிர
௯ ஒன்பதாயிர
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
௦ சுழியம்
0 சுழியம்
Loading