From 020e124bf879fc7a361482f9d0584eb17c09ab6d Mon Sep 17 00:00:00 2001 From: ramkumar-pacewisdom Date: Sun, 26 Jul 2026 19:03:41 +0530 Subject: [PATCH] normalisation added --- .python-version | 2 +- chatbot/translate/ai4Bharat/speech_to_text.py | 33 +++++++++++++++++-- pyproject.toml | 6 ++-- requirement.txt | 23 +++++++++---- requirements.in | 5 +++ 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/.python-version b/.python-version index 56d91d353..9ac380418 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10.12 +3.11.5 diff --git a/chatbot/translate/ai4Bharat/speech_to_text.py b/chatbot/translate/ai4Bharat/speech_to_text.py index b752148da..ce4b8953d 100644 --- a/chatbot/translate/ai4Bharat/speech_to_text.py +++ b/chatbot/translate/ai4Bharat/speech_to_text.py @@ -5,7 +5,9 @@ import json_repair import concurrent.futures import logging +from functools import lru_cache from chatbot.translate.base.speech_to_text import split_audio +from indic_itn import IndicITN, get_supported_languages logger = logging.getLogger('django') @@ -15,6 +17,29 @@ ai4bharat_authorization = os.getenv("BHASHANI_AUTHORIZATION") +@lru_cache(maxsize=1) +def _get_supported_languages_set(): + """Cache the supported languages set to avoid disk I/O on every call.""" + return set(get_supported_languages()) + + +@lru_cache(maxsize=16) +def _get_itn_instance(lang_code: str): + """Cache IndicITN normalizer instances per language.""" + return IndicITN(lang=lang_code) + + +def apply_itn(text: str, source_language: str) -> str: + if not text: + return text + try: + if source_language in _get_supported_languages_set(): + return _get_itn_instance(source_language).normalize(text) + except Exception as itn_err: + logger.error("Error during ITN normalization for language '%s': %s", source_language, itn_err, exc_info=True) + return text + + def transcribe_single_chunk(chunk_number, chunk, audio_format, source_language, voice_provider): b64_chunk = base64.b64encode(chunk).decode('utf-8') response = ai4bharat_speech_text( @@ -49,8 +74,10 @@ def transcribe_ai4bharat_multiple_chunks(voice_provider, base64_audio_file, sour transcripts = [future.result() for future in concurrent.futures.as_completed(futures)] transcripts.sort() - transcript = " ".join(content for _, content in transcripts) - return {'status': 200, 'content': transcript} + raw_transcript = " ".join(content for _, content in transcripts) + print("final transcript: ", raw_transcript) + final_transcript = apply_itn(raw_transcript, source_language) + return {'status': 200, 'content': final_transcript} except Exception as e: logger.error('Error processing file: %s', e, exc_info=True) @@ -138,4 +165,4 @@ def ai4bharat_speech_text(voice_provider, base64, audio_format, source_language) return { 'status': 500, 'content': str(e) - } + } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5ce454135..51d73473a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "shikshalokam-mohini-service" version = "0.1.0" description = "Add your description here" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ "asgiref>=3.11.0", "beautifulsoup4>=4.14.3", @@ -35,6 +35,7 @@ dependencies = [ "django-tailwind>=4.2.0", "djangorestframework>=3.16.1", "djangorestframework-simplejwt>=5.5.1", + "gevent>=26.4.0", "google-cloud-speech>=2.36.0", "google-cloud-texttospeech>=2.34.0", "google-cloud-translate>=3.24.0", @@ -82,9 +83,10 @@ dependencies = [ "tqdm>=4.67.1", "uuid>=1.30", "uvicorn>=0.40.0", + "indic-itn==0.2.1", ] [dependency-groups] dev = [ "ruff>=0.15.0", -] +] \ No newline at end of file diff --git a/requirement.txt b/requirement.txt index e3ca3e13a..fa7749bc4 100644 --- a/requirement.txt +++ b/requirement.txt @@ -7,13 +7,11 @@ annotated-doc==0.0.4 annotated-types==0.7.0 anyio==4.12.1 asgiref==3.11.0 -async-timeout==5.0.1 attrs==25.4.0 auth0-python==4.13.0 autobahn==24.4.2 automat==25.4.16 backoff==2.2.1 -backports-asyncio-runner==1.2.0 banks==2.3.0 bcrypt==5.0.0 beautifulsoup4==4.14.3 @@ -72,7 +70,6 @@ docx2txt==0.9 durationpy==0.10 embedchain==0.0.18 et-xmlfile==2.0.0 -exceptiongroup==1.3.1 execnet==2.1.2 fastapi==0.128.0 fastuuid==0.14.0 @@ -118,6 +115,8 @@ idna==3.11 importlib-metadata==8.7.1 importlib-resources==6.5.2 incremental==24.11.0 +indic-itn==0.2.1 + # via -r requirements.in iniconfig==2.3.0 instructor==1.14.5 itypes==1.2.0 @@ -173,7 +172,20 @@ mypy-extensions==1.1.0 nest-asyncio==1.6.0 networkx==3.4.2 nltk==3.9.2 -numpy==2.2.6 +numpy==1.26.4 + # via + # -r requirements.in + # chromadb + # datasets + # lancedb + # llama-index-core + # onnxruntime + # pandas + # qdrant-client + # scikit-learn + # scipy + # sentence-transformers + # transformers oauthlib==3.3.1 onnxruntime==1.23.2 openai==2.16.0 @@ -195,7 +207,6 @@ pdfminer-six==20251230 pdfplumber==0.11.9 pillow==12.1.0 pillow-heif==1.2.0 -pip==24.2 platformdirs==4.5.1 pluggy==1.6.0 portalocker==3.2.0 @@ -272,7 +283,6 @@ tenacity==9.1.2 threadpoolctl==3.6.0 tiktoken==0.12.0 tokenizers==0.22.2 -tomli==2.4.0 torch==2.2.2 tornado==6.5.4 tqdm==4.67.1 @@ -289,7 +299,6 @@ tzlocal==5.3.1 uritemplate==4.2.0 urllib3==2.6.3 uuid-utils==0.14.0 -uv==0.9.28 uvicorn==0.40.0 uvloop==0.22.1 vine==5.1.0 diff --git a/requirements.in b/requirements.in index d96707d44..418d68bca 100644 --- a/requirements.in +++ b/requirements.in @@ -158,3 +158,8 @@ django-redis ############################ coreapi coreschema + +############################ +# Text Normalization +############################ +indic-itn==0.2.1 \ No newline at end of file