feat: Normalisation added for speech to text- #17

.python-version

@@ -1 +1 @@
-3.10.12
+3.11.5

chatbot/translate/ai4Bharat/speech_to_text.py

@@ -5,7 +5,9 @@
 import json_repair
 import concurrent.futures
 import logging
+from functools import lru_cache
 from chatbot.translate.base.speech_to_text import split_audio
+from indic_itn import IndicITN, get_supported_languages
 
 
 logger = logging.getLogger('django')
@@ -15,6 +17,29 @@
 ai4bharat_authorization = os.getenv("BHASHANI_AUTHORIZATION")
 
 
+@lru_cache(maxsize=1)
+def _get_supported_languages_set():
+    """Cache the supported languages set to avoid disk I/O on every call."""
+    return set(get_supported_languages())
+
+
+@lru_cache(maxsize=16)
+def _get_itn_instance(lang_code: str):
+    """Cache IndicITN normalizer instances per language."""
+    return IndicITN(lang=lang_code)
+
+
+def apply_itn(text: str, source_language: str) -> str:
+    if not text:
+        return text
+    try:
+        if source_language in _get_supported_languages_set():
+            return _get_itn_instance(source_language).normalize(text)
+    except Exception as itn_err:
+        logger.error("Error during ITN normalization for language '%s': %s", source_language, itn_err, exc_info=True)
+    return text
+
+
 def transcribe_single_chunk(chunk_number, chunk, audio_format, source_language, voice_provider):
     b64_chunk = base64.b64encode(chunk).decode('utf-8')
     response = ai4bharat_speech_text(
@@ -49,8 +74,10 @@ def transcribe_ai4bharat_multiple_chunks(voice_provider, base64_audio_file, sour
             transcripts = [future.result() for future in concurrent.futures.as_completed(futures)]
             transcripts.sort()
 
-        transcript = " ".join(content for _, content in transcripts)
-        return {'status': 200, 'content': transcript}
+        raw_transcript = " ".join(content for _, content in transcripts)
+        print("final transcript: ", raw_transcript)
+        final_transcript = apply_itn(raw_transcript, source_language)
+        return {'status': 200, 'content': final_transcript}
 
     except Exception as e:
         logger.error('Error processing file: %s', e, exc_info=True)
@@ -138,4 +165,4 @@ def ai4bharat_speech_text(voice_provider, base64, audio_format, source_language)
         return {
             'status': 500,
             'content': str(e)
-        }
+        }

pyproject.toml

@@ -3,7 +3,7 @@ name = "shikshalokam-mohini-service"
 version = "0.1.0"
 description = "Add your description here"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "asgiref>=3.11.0",
     "beautifulsoup4>=4.14.3",
@@ -35,6 +35,7 @@ dependencies = [
     "django-tailwind>=4.2.0",
     "djangorestframework>=3.16.1",
     "djangorestframework-simplejwt>=5.5.1",
+    "gevent>=26.4.0",
     "google-cloud-speech>=2.36.0",
     "google-cloud-texttospeech>=2.34.0",
     "google-cloud-translate>=3.24.0",
@@ -82,9 +83,10 @@ dependencies = [
     "tqdm>=4.67.1",
     "uuid>=1.30",
     "uvicorn>=0.40.0",
+    "indic-itn==0.2.1",
 ]
 
 [dependency-groups]
 dev = [
     "ruff>=0.15.0",
-]
+]

requirement.txt

@@ -7,13 +7,11 @@ annotated-doc==0.0.4
 annotated-types==0.7.0
 anyio==4.12.1
 asgiref==3.11.0
-async-timeout==5.0.1
 attrs==25.4.0
 auth0-python==4.13.0
 autobahn==24.4.2
 automat==25.4.16
 backoff==2.2.1
-backports-asyncio-runner==1.2.0
 banks==2.3.0
 bcrypt==5.0.0
 beautifulsoup4==4.14.3
@@ -72,7 +70,6 @@ docx2txt==0.9
 durationpy==0.10
 embedchain==0.0.18
 et-xmlfile==2.0.0
-exceptiongroup==1.3.1
 execnet==2.1.2
 fastapi==0.128.0
 fastuuid==0.14.0
@@ -118,6 +115,8 @@ idna==3.11
 importlib-metadata==8.7.1
 importlib-resources==6.5.2
 incremental==24.11.0
+indic-itn==0.2.1
+    # via -r requirements.in
 iniconfig==2.3.0
 instructor==1.14.5
 itypes==1.2.0
@@ -173,7 +172,20 @@ mypy-extensions==1.1.0
 nest-asyncio==1.6.0
 networkx==3.4.2
 nltk==3.9.2
-numpy==2.2.6
+numpy==1.26.4
+    # via
+    #   -r requirements.in
+    #   chromadb
+    #   datasets
+    #   lancedb
+    #   llama-index-core
+    #   onnxruntime
+    #   pandas
+    #   qdrant-client
+    #   scikit-learn
+    #   scipy
+    #   sentence-transformers
+    #   transformers
 oauthlib==3.3.1
 onnxruntime==1.23.2
 openai==2.16.0
@@ -195,7 +207,6 @@ pdfminer-six==20251230
 pdfplumber==0.11.9
 pillow==12.1.0
 pillow-heif==1.2.0
-pip==24.2
 platformdirs==4.5.1
 pluggy==1.6.0
 portalocker==3.2.0
@@ -272,7 +283,6 @@ tenacity==9.1.2
 threadpoolctl==3.6.0
 tiktoken==0.12.0
 tokenizers==0.22.2
-tomli==2.4.0
 torch==2.2.2
 tornado==6.5.4
 tqdm==4.67.1
@@ -289,7 +299,6 @@ tzlocal==5.3.1
 uritemplate==4.2.0
 urllib3==2.6.3
 uuid-utils==0.14.0
-uv==0.9.28
 uvicorn==0.40.0
 uvloop==0.22.1
 vine==5.1.0

requirements.in

@@ -158,3 +158,8 @@ django-redis
 ############################
 coreapi
 coreschema
+
+############################
+# Text Normalization
+############################
+indic-itn==0.2.1