From 3778505b4f8976e28ee758f53f327baf7a03eb67 Mon Sep 17 00:00:00 2001 From: CodingSelim Date: Sat, 11 Jul 2026 11:45:24 +0300 Subject: [PATCH] fix dediac_hsb stripping the alef-wasla letter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HSB_DIAC_CHARSET had Ä (U+00C4) in it, but that's the HSB encoding of alef-wasla (U+0671) which is a letter, not a diacritic. dediac_hsb deletes every char in the diac set so it was silently dropping alef-wasla from valid hsb text. derived the real diac set from the ar2hsb charmap to confirm, it's exactly the current set minus Ä. added test_dediac.py (there wasn't one), the hsb cases fail before and pass after. --- camel_tools/utils/charsets.py | 2 +- tests/test_dediac.py | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/test_dediac.py diff --git a/camel_tools/utils/charsets.py b/camel_tools/utils/charsets.py index 1df74d5..a06602c 100644 --- a/camel_tools/utils/charsets.py +++ b/camel_tools/utils/charsets.py @@ -110,5 +110,5 @@ u'\u00c2\u00c4\u00e1\u00f0\u00fd\u0100\u0102' u'\u010e\u0127\u0161\u0175\u0177\u03b3\u03b8' u'\u03c2') -HSB_DIAC_CHARSET = frozenset(u'.aiu~\u00c4\u00e1\u00e3\u0129\u0169') +HSB_DIAC_CHARSET = frozenset(u'.aiu~\u00e1\u00e3\u0129\u0169') HSB_CHARSET = HSB_LETTERS_CHARSET | HSB_DIAC_CHARSET diff --git a/tests/test_dediac.py b/tests/test_dediac.py new file mode 100644 index 0000000..1044ca9 --- /dev/null +++ b/tests/test_dediac.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# MIT License +# +# Copyright 2018-2026 New York University Abu Dhabi +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Tests for camel_tools.utils.dediac +""" + +from camel_tools.utils.dediac import dediac_ar, dediac_bw, dediac_safebw +from camel_tools.utils.dediac import dediac_xmlbw, dediac_hsb + + +class TestDediacAr(object): + """Test class for dediac_ar. + """ + + def test_dediac_ar_empty(self): + assert dediac_ar(u'') == u'' + + def test_dediac_ar_removes_harakat(self): + # muHamamd with kasra/fatha/shadda/damma -> bare letters + assert dediac_ar(u'مُحَمَّد' + u'ٌ') == u'محمد' + + def test_dediac_ar_keeps_alef_wasla(self): + # dagger alef removed, alef-wasla (a letter) kept + assert dediac_ar(u'ٱلَم') == \ + u'ٱلم' + + +class TestDediacBw(object): + """Test class for dediac_bw. + """ + + def test_dediac_bw_empty(self): + assert dediac_bw(u'') == u'' + + def test_dediac_bw_removes_harakat(self): + assert dediac_bw(u'muHam~ada') == u'mHmd' + + +class TestDediacHsb(object): + """Test class for dediac_hsb. + """ + + def test_dediac_hsb_empty(self): + assert dediac_hsb(u'') == u'' + + def test_dediac_hsb_removes_harakat(self): + # fatha 'a' is a diacritic and should be removed + assert dediac_hsb(u'Äalm') == u'Älm' + + def test_dediac_hsb_keeps_alef_wasla(self): + # Alef-Wasla (Ä in HSB) is a letter, it must NOT be stripped + assert u'Ä' in dediac_hsb(u'Äalm')