diff --git a/uts39/bugfix_test.go b/uts39/bugfix_test.go new file mode 100644 index 0000000..70226e2 --- /dev/null +++ b/uts39/bugfix_test.go @@ -0,0 +1,182 @@ +package uts39 + +import ( + "strings" + "testing" +) + +// TestSkeletonCaseFold covers Bug B: the Skeleton algorithm must apply +// Unicode "full" case folding (CaseFolding.txt status C+F) rather than the +// previously used strings.ToLower, which cannot handle expansions like +// "ß" → "ss" or "ffi" → "ffi". +func TestSkeletonCaseFold(t *testing.T) { + // "ß" (U+00DF LATIN SMALL LETTER SHARP S) full-case-folds to "ss"; once + // folded it should produce the same skeleton as the literal ASCII "ss". + t.Run("sharp s folds to ss", func(t *testing.T) { + skelSharp := Skeleton("ß") + skelSS := Skeleton("ss") + if skelSharp != "ss" { + t.Errorf("Skeleton(%q) = %q, want %q", "ß", skelSharp, "ss") + } + if skelSharp != skelSS { + t.Errorf("Skeleton(%q) (%q) != Skeleton(%q) (%q)", + "ß", skelSharp, "ss", skelSS) + } + if !AreConfusable("ß", "ss") { + t.Errorf("AreConfusable(%q, %q) = false, want true", "ß", "ss") + } + // And uppercase "SS" should also match after case folding. + if !AreConfusable("ß", "SS") { + t.Errorf("AreConfusable(%q, %q) = false, want true", "ß", "SS") + } + }) + + // "ffi" (U+FB03 LATIN SMALL LIGATURE FFI) full-case-folds via NFKD to "ffi". + // This also exercises the NFKD step: the ligature decomposes to "ffi" + // under compatibility decomposition. With proper full folding the result + // should equal Skeleton("ffi"). + t.Run("ffi ligature folds to ffi", func(t *testing.T) { + skelLig := Skeleton("\uFB03") // ffi + skelFFI := Skeleton("ffi") + if skelLig != skelFFI { + t.Errorf("Skeleton(%q) (%q) != Skeleton(%q) (%q)", + "\uFB03", skelLig, "ffi", skelFFI) + } + }) + + // "İ" (U+0130 LATIN CAPITAL LETTER I WITH DOT ABOVE) full-case-folds + // (non-Turkic) to "i\u0307" (LATIN SMALL LETTER I + COMBINING DOT ABOVE), + // not to plain "i". This test confirms the folding actually runs (versus + // the old strings.ToLower path which produced just "i"). + t.Run("Turkish-I folds via full case folding", func(t *testing.T) { + skel := Skeleton("İ") + // Skeleton must contain the COMBINING DOT ABOVE that full case + // folding introduces; if it does not, we are still using ToLower. + if !strings.ContainsRune(skel, '\u0307') { + t.Errorf("Skeleton(%q) = %q; expected to contain U+0307 "+ + "COMBINING DOT ABOVE from full case folding", "İ", skel) + } + // Per UTS #39 §4 with full (non-Turkic) folding, "İ" and "I" do not + // produce identical skeletons. This is the documented spec behavior; + // we assert it explicitly so a future change does not silently flip it. + if AreConfusable("I", "İ") { + t.Errorf("AreConfusable(%q, %q) = true; expected false under "+ + "full (non-Turkic) case folding per UTS #39 §4", + "I", "İ") + } + }) +} + +// TestCaseFoldHelper exercises the unexported caseFold helper directly to +// verify its behavior independently of the surrounding Skeleton pipeline. +func TestCaseFoldHelper(t *testing.T) { + tests := []struct { + in, want string + }{ + {"", ""}, + {"abc", "abc"}, // ASCII fast-path no-op + {"ABC", "abc"}, // ASCII fast-path fold + {"Hello", "hello"}, // mixed-case ASCII + {"ß", "ss"}, // expansion via F mapping + {"İ", "i\u0307"}, // F mapping with combining mark + {"\u00DF\u00DF", "ssss"}, + {"\u0130A", "i\u0307a"}, // mixed + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + got := caseFold(tt.in) + if got != tt.want { + t.Errorf("caseFold(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +// TestSkeletonIdempotent verifies that Skeleton is idempotent on its output, +// which is part of how Bug C (the fixed iteration cap) is justified: if the +// transform is idempotent, looping to a fixed point is well-defined. +func TestSkeletonIdempotent(t *testing.T) { + inputs := []string{ + "", + "hello", + "HELLO", + "ß", + "İ", + "paypal", + "p\u0430ypal", // Cyrillic а + "\u0455\u0441\u043E\u0440\u0435", // Cyrillic ѕсоре + "café", + "cafe\u0301", + "\uFB03ce", // ligature ffi + "ce" + "hello世界", + "hello漢字ㅎ", + } + for _, in := range inputs { + t.Run(in, func(t *testing.T) { + once := Skeleton(in) + twice := Skeleton(once) + if once != twice { + t.Errorf("Skeleton not idempotent for %q: first=%q, second=%q", + in, once, twice) + } + }) + } +} + +// TestGetRestrictionLevelTable1 covers each restriction level in UAX #39 +// §5.2 Table 1 with at least one representative input. This is intentionally +// duplicative with TestGetRestrictionLevel (in uts39_test.go) so the levels +// remain pinned even if the existing tests drift. +func TestGetRestrictionLevelTable1(t *testing.T) { + tests := []struct { + name string + input string + want RestrictionLevel + }{ + // ASCIIOnly + {"ASCII only", "hello", ASCIIOnly}, + {"ASCII with digits", "user_123", ASCIIOnly}, + + // SingleScript + {"Single Latin", "café", SingleScript}, + {"Single Cyrillic", "привет", SingleScript}, + {"Single Han", "你好", SingleScript}, + {"Single Greek", "γεια", SingleScript}, + {"Single Arabic", "مرحبا", SingleScript}, + + // HighlyRestrictive: Latn + Jpan / Hanb / Kore + {"Latin + Han (Latn+Jpan subset)", "hello世界", HighlyRestrictive}, + {"Latin + Han + Hiragana", "hello漢字ひらがな", HighlyRestrictive}, + {"Latin + Han + Katakana", "helloカタカナ漢", HighlyRestrictive}, + {"Latin + Han + Hangul", "hello漢字ㅎ", HighlyRestrictive}, + {"Latin + Han + Bopomofo", "hello漢ㄅ", HighlyRestrictive}, + {"Latin + Han + Hiragana + Katakana", "hi漢ひカ", HighlyRestrictive}, + + // ModeratelyRestrictive: Latin + exactly one other non-{Cyrillic,Greek} + // recommended script. + {"Latin + Arabic", "helloمرحبا", ModeratelyRestrictive}, + {"Latin + Hebrew", "helloשלום", ModeratelyRestrictive}, + {"Latin + Devanagari", "helloनमस्ते", ModeratelyRestrictive}, + {"Latin + Thai", "helloสวัสดี", ModeratelyRestrictive}, + + // MinimallyRestrictive: catch-all for other multi-script combinations. + {"Latin + Cyrillic", "hello мир", MinimallyRestrictive}, + {"Latin + Greek", "helloαβ", MinimallyRestrictive}, + {"Latin + Cyrillic + Greek", "helloмαβ", MinimallyRestrictive}, + {"Latin + Arabic + Hebrew", + "helloمرحباשלום", MinimallyRestrictive}, + + // Unrestricted: empty fallback. + {"Empty string", "", Unrestricted}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetRestrictionLevel(tt.input) + if got != tt.want { + t.Errorf("GetRestrictionLevel(%q) = %v, want %v", + tt.input, got, tt.want) + } + }) + } +} diff --git a/uts39/casefold_data.go b/uts39/casefold_data.go new file mode 100644 index 0000000..eee7a14 --- /dev/null +++ b/uts39/casefold_data.go @@ -0,0 +1,1604 @@ +// Code generated by generate_casefold.go. DO NOT EDIT. +// Source: https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt +// Unicode version: 17.0.0 +// Includes status C (common) and F (full) mappings only, +// per UTS #39's "full case folding" requirement. + +package uts39 + +// caseFoldEntry represents a case folding mapping from a single +// source code point to (possibly multiple) target code points. +type caseFoldEntry struct { + source rune + target string +} + +// caseFoldData contains the full case folding mappings (C + F) +// from CaseFolding.txt. Sorted by source for binary search. +var caseFoldData = []caseFoldEntry{ + {0x0041, "a"}, // (C) + {0x0042, "b"}, // (C) + {0x0043, "c"}, // (C) + {0x0044, "d"}, // (C) + {0x0045, "e"}, // (C) + {0x0046, "f"}, // (C) + {0x0047, "g"}, // (C) + {0x0048, "h"}, // (C) + {0x0049, "i"}, // (C) + {0x004A, "j"}, // (C) + {0x004B, "k"}, // (C) + {0x004C, "l"}, // (C) + {0x004D, "m"}, // (C) + {0x004E, "n"}, // (C) + {0x004F, "o"}, // (C) + {0x0050, "p"}, // (C) + {0x0051, "q"}, // (C) + {0x0052, "r"}, // (C) + {0x0053, "s"}, // (C) + {0x0054, "t"}, // (C) + {0x0055, "u"}, // (C) + {0x0056, "v"}, // (C) + {0x0057, "w"}, // (C) + {0x0058, "x"}, // (C) + {0x0059, "y"}, // (C) + {0x005A, "z"}, // (C) + {0x00B5, "\u03bc"}, // (C) + {0x00C0, "\u00e0"}, // (C) + {0x00C1, "\u00e1"}, // (C) + {0x00C2, "\u00e2"}, // (C) + {0x00C3, "\u00e3"}, // (C) + {0x00C4, "\u00e4"}, // (C) + {0x00C5, "\u00e5"}, // (C) + {0x00C6, "\u00e6"}, // (C) + {0x00C7, "\u00e7"}, // (C) + {0x00C8, "\u00e8"}, // (C) + {0x00C9, "\u00e9"}, // (C) + {0x00CA, "\u00ea"}, // (C) + {0x00CB, "\u00eb"}, // (C) + {0x00CC, "\u00ec"}, // (C) + {0x00CD, "\u00ed"}, // (C) + {0x00CE, "\u00ee"}, // (C) + {0x00CF, "\u00ef"}, // (C) + {0x00D0, "\u00f0"}, // (C) + {0x00D1, "\u00f1"}, // (C) + {0x00D2, "\u00f2"}, // (C) + {0x00D3, "\u00f3"}, // (C) + {0x00D4, "\u00f4"}, // (C) + {0x00D5, "\u00f5"}, // (C) + {0x00D6, "\u00f6"}, // (C) + {0x00D8, "\u00f8"}, // (C) + {0x00D9, "\u00f9"}, // (C) + {0x00DA, "\u00fa"}, // (C) + {0x00DB, "\u00fb"}, // (C) + {0x00DC, "\u00fc"}, // (C) + {0x00DD, "\u00fd"}, // (C) + {0x00DE, "\u00fe"}, // (C) + {0x00DF, "ss"}, // (F) + {0x0100, "\u0101"}, // (C) + {0x0102, "\u0103"}, // (C) + {0x0104, "\u0105"}, // (C) + {0x0106, "\u0107"}, // (C) + {0x0108, "\u0109"}, // (C) + {0x010A, "\u010b"}, // (C) + {0x010C, "\u010d"}, // (C) + {0x010E, "\u010f"}, // (C) + {0x0110, "\u0111"}, // (C) + {0x0112, "\u0113"}, // (C) + {0x0114, "\u0115"}, // (C) + {0x0116, "\u0117"}, // (C) + {0x0118, "\u0119"}, // (C) + {0x011A, "\u011b"}, // (C) + {0x011C, "\u011d"}, // (C) + {0x011E, "\u011f"}, // (C) + {0x0120, "\u0121"}, // (C) + {0x0122, "\u0123"}, // (C) + {0x0124, "\u0125"}, // (C) + {0x0126, "\u0127"}, // (C) + {0x0128, "\u0129"}, // (C) + {0x012A, "\u012b"}, // (C) + {0x012C, "\u012d"}, // (C) + {0x012E, "\u012f"}, // (C) + {0x0130, "i\u0307"}, // (F) + {0x0132, "\u0133"}, // (C) + {0x0134, "\u0135"}, // (C) + {0x0136, "\u0137"}, // (C) + {0x0139, "\u013a"}, // (C) + {0x013B, "\u013c"}, // (C) + {0x013D, "\u013e"}, // (C) + {0x013F, "\u0140"}, // (C) + {0x0141, "\u0142"}, // (C) + {0x0143, "\u0144"}, // (C) + {0x0145, "\u0146"}, // (C) + {0x0147, "\u0148"}, // (C) + {0x0149, "\u02bcn"}, // (F) + {0x014A, "\u014b"}, // (C) + {0x014C, "\u014d"}, // (C) + {0x014E, "\u014f"}, // (C) + {0x0150, "\u0151"}, // (C) + {0x0152, "\u0153"}, // (C) + {0x0154, "\u0155"}, // (C) + {0x0156, "\u0157"}, // (C) + {0x0158, "\u0159"}, // (C) + {0x015A, "\u015b"}, // (C) + {0x015C, "\u015d"}, // (C) + {0x015E, "\u015f"}, // (C) + {0x0160, "\u0161"}, // (C) + {0x0162, "\u0163"}, // (C) + {0x0164, "\u0165"}, // (C) + {0x0166, "\u0167"}, // (C) + {0x0168, "\u0169"}, // (C) + {0x016A, "\u016b"}, // (C) + {0x016C, "\u016d"}, // (C) + {0x016E, "\u016f"}, // (C) + {0x0170, "\u0171"}, // (C) + {0x0172, "\u0173"}, // (C) + {0x0174, "\u0175"}, // (C) + {0x0176, "\u0177"}, // (C) + {0x0178, "\u00ff"}, // (C) + {0x0179, "\u017a"}, // (C) + {0x017B, "\u017c"}, // (C) + {0x017D, "\u017e"}, // (C) + {0x017F, "s"}, // (C) + {0x0181, "\u0253"}, // (C) + {0x0182, "\u0183"}, // (C) + {0x0184, "\u0185"}, // (C) + {0x0186, "\u0254"}, // (C) + {0x0187, "\u0188"}, // (C) + {0x0189, "\u0256"}, // (C) + {0x018A, "\u0257"}, // (C) + {0x018B, "\u018c"}, // (C) + {0x018E, "\u01dd"}, // (C) + {0x018F, "\u0259"}, // (C) + {0x0190, "\u025b"}, // (C) + {0x0191, "\u0192"}, // (C) + {0x0193, "\u0260"}, // (C) + {0x0194, "\u0263"}, // (C) + {0x0196, "\u0269"}, // (C) + {0x0197, "\u0268"}, // (C) + {0x0198, "\u0199"}, // (C) + {0x019C, "\u026f"}, // (C) + {0x019D, "\u0272"}, // (C) + {0x019F, "\u0275"}, // (C) + {0x01A0, "\u01a1"}, // (C) + {0x01A2, "\u01a3"}, // (C) + {0x01A4, "\u01a5"}, // (C) + {0x01A6, "\u0280"}, // (C) + {0x01A7, "\u01a8"}, // (C) + {0x01A9, "\u0283"}, // (C) + {0x01AC, "\u01ad"}, // (C) + {0x01AE, "\u0288"}, // (C) + {0x01AF, "\u01b0"}, // (C) + {0x01B1, "\u028a"}, // (C) + {0x01B2, "\u028b"}, // (C) + {0x01B3, "\u01b4"}, // (C) + {0x01B5, "\u01b6"}, // (C) + {0x01B7, "\u0292"}, // (C) + {0x01B8, "\u01b9"}, // (C) + {0x01BC, "\u01bd"}, // (C) + {0x01C4, "\u01c6"}, // (C) + {0x01C5, "\u01c6"}, // (C) + {0x01C7, "\u01c9"}, // (C) + {0x01C8, "\u01c9"}, // (C) + {0x01CA, "\u01cc"}, // (C) + {0x01CB, "\u01cc"}, // (C) + {0x01CD, "\u01ce"}, // (C) + {0x01CF, "\u01d0"}, // (C) + {0x01D1, "\u01d2"}, // (C) + {0x01D3, "\u01d4"}, // (C) + {0x01D5, "\u01d6"}, // (C) + {0x01D7, "\u01d8"}, // (C) + {0x01D9, "\u01da"}, // (C) + {0x01DB, "\u01dc"}, // (C) + {0x01DE, "\u01df"}, // (C) + {0x01E0, "\u01e1"}, // (C) + {0x01E2, "\u01e3"}, // (C) + {0x01E4, "\u01e5"}, // (C) + {0x01E6, "\u01e7"}, // (C) + {0x01E8, "\u01e9"}, // (C) + {0x01EA, "\u01eb"}, // (C) + {0x01EC, "\u01ed"}, // (C) + {0x01EE, "\u01ef"}, // (C) + {0x01F0, "j\u030c"}, // (F) + {0x01F1, "\u01f3"}, // (C) + {0x01F2, "\u01f3"}, // (C) + {0x01F4, "\u01f5"}, // (C) + {0x01F6, "\u0195"}, // (C) + {0x01F7, "\u01bf"}, // (C) + {0x01F8, "\u01f9"}, // (C) + {0x01FA, "\u01fb"}, // (C) + {0x01FC, "\u01fd"}, // (C) + {0x01FE, "\u01ff"}, // (C) + {0x0200, "\u0201"}, // (C) + {0x0202, "\u0203"}, // (C) + {0x0204, "\u0205"}, // (C) + {0x0206, "\u0207"}, // (C) + {0x0208, "\u0209"}, // (C) + {0x020A, "\u020b"}, // (C) + {0x020C, "\u020d"}, // (C) + {0x020E, "\u020f"}, // (C) + {0x0210, "\u0211"}, // (C) + {0x0212, "\u0213"}, // (C) + {0x0214, "\u0215"}, // (C) + {0x0216, "\u0217"}, // (C) + {0x0218, "\u0219"}, // (C) + {0x021A, "\u021b"}, // (C) + {0x021C, "\u021d"}, // (C) + {0x021E, "\u021f"}, // (C) + {0x0220, "\u019e"}, // (C) + {0x0222, "\u0223"}, // (C) + {0x0224, "\u0225"}, // (C) + {0x0226, "\u0227"}, // (C) + {0x0228, "\u0229"}, // (C) + {0x022A, "\u022b"}, // (C) + {0x022C, "\u022d"}, // (C) + {0x022E, "\u022f"}, // (C) + {0x0230, "\u0231"}, // (C) + {0x0232, "\u0233"}, // (C) + {0x023A, "\u2c65"}, // (C) + {0x023B, "\u023c"}, // (C) + {0x023D, "\u019a"}, // (C) + {0x023E, "\u2c66"}, // (C) + {0x0241, "\u0242"}, // (C) + {0x0243, "\u0180"}, // (C) + {0x0244, "\u0289"}, // (C) + {0x0245, "\u028c"}, // (C) + {0x0246, "\u0247"}, // (C) + {0x0248, "\u0249"}, // (C) + {0x024A, "\u024b"}, // (C) + {0x024C, "\u024d"}, // (C) + {0x024E, "\u024f"}, // (C) + {0x0345, "\u03b9"}, // (C) + {0x0370, "\u0371"}, // (C) + {0x0372, "\u0373"}, // (C) + {0x0376, "\u0377"}, // (C) + {0x037F, "\u03f3"}, // (C) + {0x0386, "\u03ac"}, // (C) + {0x0388, "\u03ad"}, // (C) + {0x0389, "\u03ae"}, // (C) + {0x038A, "\u03af"}, // (C) + {0x038C, "\u03cc"}, // (C) + {0x038E, "\u03cd"}, // (C) + {0x038F, "\u03ce"}, // (C) + {0x0390, "\u03b9\u0308\u0301"}, // (F) + {0x0391, "\u03b1"}, // (C) + {0x0392, "\u03b2"}, // (C) + {0x0393, "\u03b3"}, // (C) + {0x0394, "\u03b4"}, // (C) + {0x0395, "\u03b5"}, // (C) + {0x0396, "\u03b6"}, // (C) + {0x0397, "\u03b7"}, // (C) + {0x0398, "\u03b8"}, // (C) + {0x0399, "\u03b9"}, // (C) + {0x039A, "\u03ba"}, // (C) + {0x039B, "\u03bb"}, // (C) + {0x039C, "\u03bc"}, // (C) + {0x039D, "\u03bd"}, // (C) + {0x039E, "\u03be"}, // (C) + {0x039F, "\u03bf"}, // (C) + {0x03A0, "\u03c0"}, // (C) + {0x03A1, "\u03c1"}, // (C) + {0x03A3, "\u03c3"}, // (C) + {0x03A4, "\u03c4"}, // (C) + {0x03A5, "\u03c5"}, // (C) + {0x03A6, "\u03c6"}, // (C) + {0x03A7, "\u03c7"}, // (C) + {0x03A8, "\u03c8"}, // (C) + {0x03A9, "\u03c9"}, // (C) + {0x03AA, "\u03ca"}, // (C) + {0x03AB, "\u03cb"}, // (C) + {0x03B0, "\u03c5\u0308\u0301"}, // (F) + {0x03C2, "\u03c3"}, // (C) + {0x03CF, "\u03d7"}, // (C) + {0x03D0, "\u03b2"}, // (C) + {0x03D1, "\u03b8"}, // (C) + {0x03D5, "\u03c6"}, // (C) + {0x03D6, "\u03c0"}, // (C) + {0x03D8, "\u03d9"}, // (C) + {0x03DA, "\u03db"}, // (C) + {0x03DC, "\u03dd"}, // (C) + {0x03DE, "\u03df"}, // (C) + {0x03E0, "\u03e1"}, // (C) + {0x03E2, "\u03e3"}, // (C) + {0x03E4, "\u03e5"}, // (C) + {0x03E6, "\u03e7"}, // (C) + {0x03E8, "\u03e9"}, // (C) + {0x03EA, "\u03eb"}, // (C) + {0x03EC, "\u03ed"}, // (C) + {0x03EE, "\u03ef"}, // (C) + {0x03F0, "\u03ba"}, // (C) + {0x03F1, "\u03c1"}, // (C) + {0x03F4, "\u03b8"}, // (C) + {0x03F5, "\u03b5"}, // (C) + {0x03F7, "\u03f8"}, // (C) + {0x03F9, "\u03f2"}, // (C) + {0x03FA, "\u03fb"}, // (C) + {0x03FD, "\u037b"}, // (C) + {0x03FE, "\u037c"}, // (C) + {0x03FF, "\u037d"}, // (C) + {0x0400, "\u0450"}, // (C) + {0x0401, "\u0451"}, // (C) + {0x0402, "\u0452"}, // (C) + {0x0403, "\u0453"}, // (C) + {0x0404, "\u0454"}, // (C) + {0x0405, "\u0455"}, // (C) + {0x0406, "\u0456"}, // (C) + {0x0407, "\u0457"}, // (C) + {0x0408, "\u0458"}, // (C) + {0x0409, "\u0459"}, // (C) + {0x040A, "\u045a"}, // (C) + {0x040B, "\u045b"}, // (C) + {0x040C, "\u045c"}, // (C) + {0x040D, "\u045d"}, // (C) + {0x040E, "\u045e"}, // (C) + {0x040F, "\u045f"}, // (C) + {0x0410, "\u0430"}, // (C) + {0x0411, "\u0431"}, // (C) + {0x0412, "\u0432"}, // (C) + {0x0413, "\u0433"}, // (C) + {0x0414, "\u0434"}, // (C) + {0x0415, "\u0435"}, // (C) + {0x0416, "\u0436"}, // (C) + {0x0417, "\u0437"}, // (C) + {0x0418, "\u0438"}, // (C) + {0x0419, "\u0439"}, // (C) + {0x041A, "\u043a"}, // (C) + {0x041B, "\u043b"}, // (C) + {0x041C, "\u043c"}, // (C) + {0x041D, "\u043d"}, // (C) + {0x041E, "\u043e"}, // (C) + {0x041F, "\u043f"}, // (C) + {0x0420, "\u0440"}, // (C) + {0x0421, "\u0441"}, // (C) + {0x0422, "\u0442"}, // (C) + {0x0423, "\u0443"}, // (C) + {0x0424, "\u0444"}, // (C) + {0x0425, "\u0445"}, // (C) + {0x0426, "\u0446"}, // (C) + {0x0427, "\u0447"}, // (C) + {0x0428, "\u0448"}, // (C) + {0x0429, "\u0449"}, // (C) + {0x042A, "\u044a"}, // (C) + {0x042B, "\u044b"}, // (C) + {0x042C, "\u044c"}, // (C) + {0x042D, "\u044d"}, // (C) + {0x042E, "\u044e"}, // (C) + {0x042F, "\u044f"}, // (C) + {0x0460, "\u0461"}, // (C) + {0x0462, "\u0463"}, // (C) + {0x0464, "\u0465"}, // (C) + {0x0466, "\u0467"}, // (C) + {0x0468, "\u0469"}, // (C) + {0x046A, "\u046b"}, // (C) + {0x046C, "\u046d"}, // (C) + {0x046E, "\u046f"}, // (C) + {0x0470, "\u0471"}, // (C) + {0x0472, "\u0473"}, // (C) + {0x0474, "\u0475"}, // (C) + {0x0476, "\u0477"}, // (C) + {0x0478, "\u0479"}, // (C) + {0x047A, "\u047b"}, // (C) + {0x047C, "\u047d"}, // (C) + {0x047E, "\u047f"}, // (C) + {0x0480, "\u0481"}, // (C) + {0x048A, "\u048b"}, // (C) + {0x048C, "\u048d"}, // (C) + {0x048E, "\u048f"}, // (C) + {0x0490, "\u0491"}, // (C) + {0x0492, "\u0493"}, // (C) + {0x0494, "\u0495"}, // (C) + {0x0496, "\u0497"}, // (C) + {0x0498, "\u0499"}, // (C) + {0x049A, "\u049b"}, // (C) + {0x049C, "\u049d"}, // (C) + {0x049E, "\u049f"}, // (C) + {0x04A0, "\u04a1"}, // (C) + {0x04A2, "\u04a3"}, // (C) + {0x04A4, "\u04a5"}, // (C) + {0x04A6, "\u04a7"}, // (C) + {0x04A8, "\u04a9"}, // (C) + {0x04AA, "\u04ab"}, // (C) + {0x04AC, "\u04ad"}, // (C) + {0x04AE, "\u04af"}, // (C) + {0x04B0, "\u04b1"}, // (C) + {0x04B2, "\u04b3"}, // (C) + {0x04B4, "\u04b5"}, // (C) + {0x04B6, "\u04b7"}, // (C) + {0x04B8, "\u04b9"}, // (C) + {0x04BA, "\u04bb"}, // (C) + {0x04BC, "\u04bd"}, // (C) + {0x04BE, "\u04bf"}, // (C) + {0x04C0, "\u04cf"}, // (C) + {0x04C1, "\u04c2"}, // (C) + {0x04C3, "\u04c4"}, // (C) + {0x04C5, "\u04c6"}, // (C) + {0x04C7, "\u04c8"}, // (C) + {0x04C9, "\u04ca"}, // (C) + {0x04CB, "\u04cc"}, // (C) + {0x04CD, "\u04ce"}, // (C) + {0x04D0, "\u04d1"}, // (C) + {0x04D2, "\u04d3"}, // (C) + {0x04D4, "\u04d5"}, // (C) + {0x04D6, "\u04d7"}, // (C) + {0x04D8, "\u04d9"}, // (C) + {0x04DA, "\u04db"}, // (C) + {0x04DC, "\u04dd"}, // (C) + {0x04DE, "\u04df"}, // (C) + {0x04E0, "\u04e1"}, // (C) + {0x04E2, "\u04e3"}, // (C) + {0x04E4, "\u04e5"}, // (C) + {0x04E6, "\u04e7"}, // (C) + {0x04E8, "\u04e9"}, // (C) + {0x04EA, "\u04eb"}, // (C) + {0x04EC, "\u04ed"}, // (C) + {0x04EE, "\u04ef"}, // (C) + {0x04F0, "\u04f1"}, // (C) + {0x04F2, "\u04f3"}, // (C) + {0x04F4, "\u04f5"}, // (C) + {0x04F6, "\u04f7"}, // (C) + {0x04F8, "\u04f9"}, // (C) + {0x04FA, "\u04fb"}, // (C) + {0x04FC, "\u04fd"}, // (C) + {0x04FE, "\u04ff"}, // (C) + {0x0500, "\u0501"}, // (C) + {0x0502, "\u0503"}, // (C) + {0x0504, "\u0505"}, // (C) + {0x0506, "\u0507"}, // (C) + {0x0508, "\u0509"}, // (C) + {0x050A, "\u050b"}, // (C) + {0x050C, "\u050d"}, // (C) + {0x050E, "\u050f"}, // (C) + {0x0510, "\u0511"}, // (C) + {0x0512, "\u0513"}, // (C) + {0x0514, "\u0515"}, // (C) + {0x0516, "\u0517"}, // (C) + {0x0518, "\u0519"}, // (C) + {0x051A, "\u051b"}, // (C) + {0x051C, "\u051d"}, // (C) + {0x051E, "\u051f"}, // (C) + {0x0520, "\u0521"}, // (C) + {0x0522, "\u0523"}, // (C) + {0x0524, "\u0525"}, // (C) + {0x0526, "\u0527"}, // (C) + {0x0528, "\u0529"}, // (C) + {0x052A, "\u052b"}, // (C) + {0x052C, "\u052d"}, // (C) + {0x052E, "\u052f"}, // (C) + {0x0531, "\u0561"}, // (C) + {0x0532, "\u0562"}, // (C) + {0x0533, "\u0563"}, // (C) + {0x0534, "\u0564"}, // (C) + {0x0535, "\u0565"}, // (C) + {0x0536, "\u0566"}, // (C) + {0x0537, "\u0567"}, // (C) + {0x0538, "\u0568"}, // (C) + {0x0539, "\u0569"}, // (C) + {0x053A, "\u056a"}, // (C) + {0x053B, "\u056b"}, // (C) + {0x053C, "\u056c"}, // (C) + {0x053D, "\u056d"}, // (C) + {0x053E, "\u056e"}, // (C) + {0x053F, "\u056f"}, // (C) + {0x0540, "\u0570"}, // (C) + {0x0541, "\u0571"}, // (C) + {0x0542, "\u0572"}, // (C) + {0x0543, "\u0573"}, // (C) + {0x0544, "\u0574"}, // (C) + {0x0545, "\u0575"}, // (C) + {0x0546, "\u0576"}, // (C) + {0x0547, "\u0577"}, // (C) + {0x0548, "\u0578"}, // (C) + {0x0549, "\u0579"}, // (C) + {0x054A, "\u057a"}, // (C) + {0x054B, "\u057b"}, // (C) + {0x054C, "\u057c"}, // (C) + {0x054D, "\u057d"}, // (C) + {0x054E, "\u057e"}, // (C) + {0x054F, "\u057f"}, // (C) + {0x0550, "\u0580"}, // (C) + {0x0551, "\u0581"}, // (C) + {0x0552, "\u0582"}, // (C) + {0x0553, "\u0583"}, // (C) + {0x0554, "\u0584"}, // (C) + {0x0555, "\u0585"}, // (C) + {0x0556, "\u0586"}, // (C) + {0x0587, "\u0565\u0582"}, // (F) + {0x10A0, "\u2d00"}, // (C) + {0x10A1, "\u2d01"}, // (C) + {0x10A2, "\u2d02"}, // (C) + {0x10A3, "\u2d03"}, // (C) + {0x10A4, "\u2d04"}, // (C) + {0x10A5, "\u2d05"}, // (C) + {0x10A6, "\u2d06"}, // (C) + {0x10A7, "\u2d07"}, // (C) + {0x10A8, "\u2d08"}, // (C) + {0x10A9, "\u2d09"}, // (C) + {0x10AA, "\u2d0a"}, // (C) + {0x10AB, "\u2d0b"}, // (C) + {0x10AC, "\u2d0c"}, // (C) + {0x10AD, "\u2d0d"}, // (C) + {0x10AE, "\u2d0e"}, // (C) + {0x10AF, "\u2d0f"}, // (C) + {0x10B0, "\u2d10"}, // (C) + {0x10B1, "\u2d11"}, // (C) + {0x10B2, "\u2d12"}, // (C) + {0x10B3, "\u2d13"}, // (C) + {0x10B4, "\u2d14"}, // (C) + {0x10B5, "\u2d15"}, // (C) + {0x10B6, "\u2d16"}, // (C) + {0x10B7, "\u2d17"}, // (C) + {0x10B8, "\u2d18"}, // (C) + {0x10B9, "\u2d19"}, // (C) + {0x10BA, "\u2d1a"}, // (C) + {0x10BB, "\u2d1b"}, // (C) + {0x10BC, "\u2d1c"}, // (C) + {0x10BD, "\u2d1d"}, // (C) + {0x10BE, "\u2d1e"}, // (C) + {0x10BF, "\u2d1f"}, // (C) + {0x10C0, "\u2d20"}, // (C) + {0x10C1, "\u2d21"}, // (C) + {0x10C2, "\u2d22"}, // (C) + {0x10C3, "\u2d23"}, // (C) + {0x10C4, "\u2d24"}, // (C) + {0x10C5, "\u2d25"}, // (C) + {0x10C7, "\u2d27"}, // (C) + {0x10CD, "\u2d2d"}, // (C) + {0x13F8, "\u13f0"}, // (C) + {0x13F9, "\u13f1"}, // (C) + {0x13FA, "\u13f2"}, // (C) + {0x13FB, "\u13f3"}, // (C) + {0x13FC, "\u13f4"}, // (C) + {0x13FD, "\u13f5"}, // (C) + {0x1C80, "\u0432"}, // (C) + {0x1C81, "\u0434"}, // (C) + {0x1C82, "\u043e"}, // (C) + {0x1C83, "\u0441"}, // (C) + {0x1C84, "\u0442"}, // (C) + {0x1C85, "\u0442"}, // (C) + {0x1C86, "\u044a"}, // (C) + {0x1C87, "\u0463"}, // (C) + {0x1C88, "\ua64b"}, // (C) + {0x1C89, "\u1c8a"}, // (C) + {0x1C90, "\u10d0"}, // (C) + {0x1C91, "\u10d1"}, // (C) + {0x1C92, "\u10d2"}, // (C) + {0x1C93, "\u10d3"}, // (C) + {0x1C94, "\u10d4"}, // (C) + {0x1C95, "\u10d5"}, // (C) + {0x1C96, "\u10d6"}, // (C) + {0x1C97, "\u10d7"}, // (C) + {0x1C98, "\u10d8"}, // (C) + {0x1C99, "\u10d9"}, // (C) + {0x1C9A, "\u10da"}, // (C) + {0x1C9B, "\u10db"}, // (C) + {0x1C9C, "\u10dc"}, // (C) + {0x1C9D, "\u10dd"}, // (C) + {0x1C9E, "\u10de"}, // (C) + {0x1C9F, "\u10df"}, // (C) + {0x1CA0, "\u10e0"}, // (C) + {0x1CA1, "\u10e1"}, // (C) + {0x1CA2, "\u10e2"}, // (C) + {0x1CA3, "\u10e3"}, // (C) + {0x1CA4, "\u10e4"}, // (C) + {0x1CA5, "\u10e5"}, // (C) + {0x1CA6, "\u10e6"}, // (C) + {0x1CA7, "\u10e7"}, // (C) + {0x1CA8, "\u10e8"}, // (C) + {0x1CA9, "\u10e9"}, // (C) + {0x1CAA, "\u10ea"}, // (C) + {0x1CAB, "\u10eb"}, // (C) + {0x1CAC, "\u10ec"}, // (C) + {0x1CAD, "\u10ed"}, // (C) + {0x1CAE, "\u10ee"}, // (C) + {0x1CAF, "\u10ef"}, // (C) + {0x1CB0, "\u10f0"}, // (C) + {0x1CB1, "\u10f1"}, // (C) + {0x1CB2, "\u10f2"}, // (C) + {0x1CB3, "\u10f3"}, // (C) + {0x1CB4, "\u10f4"}, // (C) + {0x1CB5, "\u10f5"}, // (C) + {0x1CB6, "\u10f6"}, // (C) + {0x1CB7, "\u10f7"}, // (C) + {0x1CB8, "\u10f8"}, // (C) + {0x1CB9, "\u10f9"}, // (C) + {0x1CBA, "\u10fa"}, // (C) + {0x1CBD, "\u10fd"}, // (C) + {0x1CBE, "\u10fe"}, // (C) + {0x1CBF, "\u10ff"}, // (C) + {0x1E00, "\u1e01"}, // (C) + {0x1E02, "\u1e03"}, // (C) + {0x1E04, "\u1e05"}, // (C) + {0x1E06, "\u1e07"}, // (C) + {0x1E08, "\u1e09"}, // (C) + {0x1E0A, "\u1e0b"}, // (C) + {0x1E0C, "\u1e0d"}, // (C) + {0x1E0E, "\u1e0f"}, // (C) + {0x1E10, "\u1e11"}, // (C) + {0x1E12, "\u1e13"}, // (C) + {0x1E14, "\u1e15"}, // (C) + {0x1E16, "\u1e17"}, // (C) + {0x1E18, "\u1e19"}, // (C) + {0x1E1A, "\u1e1b"}, // (C) + {0x1E1C, "\u1e1d"}, // (C) + {0x1E1E, "\u1e1f"}, // (C) + {0x1E20, "\u1e21"}, // (C) + {0x1E22, "\u1e23"}, // (C) + {0x1E24, "\u1e25"}, // (C) + {0x1E26, "\u1e27"}, // (C) + {0x1E28, "\u1e29"}, // (C) + {0x1E2A, "\u1e2b"}, // (C) + {0x1E2C, "\u1e2d"}, // (C) + {0x1E2E, "\u1e2f"}, // (C) + {0x1E30, "\u1e31"}, // (C) + {0x1E32, "\u1e33"}, // (C) + {0x1E34, "\u1e35"}, // (C) + {0x1E36, "\u1e37"}, // (C) + {0x1E38, "\u1e39"}, // (C) + {0x1E3A, "\u1e3b"}, // (C) + {0x1E3C, "\u1e3d"}, // (C) + {0x1E3E, "\u1e3f"}, // (C) + {0x1E40, "\u1e41"}, // (C) + {0x1E42, "\u1e43"}, // (C) + {0x1E44, "\u1e45"}, // (C) + {0x1E46, "\u1e47"}, // (C) + {0x1E48, "\u1e49"}, // (C) + {0x1E4A, "\u1e4b"}, // (C) + {0x1E4C, "\u1e4d"}, // (C) + {0x1E4E, "\u1e4f"}, // (C) + {0x1E50, "\u1e51"}, // (C) + {0x1E52, "\u1e53"}, // (C) + {0x1E54, "\u1e55"}, // (C) + {0x1E56, "\u1e57"}, // (C) + {0x1E58, "\u1e59"}, // (C) + {0x1E5A, "\u1e5b"}, // (C) + {0x1E5C, "\u1e5d"}, // (C) + {0x1E5E, "\u1e5f"}, // (C) + {0x1E60, "\u1e61"}, // (C) + {0x1E62, "\u1e63"}, // (C) + {0x1E64, "\u1e65"}, // (C) + {0x1E66, "\u1e67"}, // (C) + {0x1E68, "\u1e69"}, // (C) + {0x1E6A, "\u1e6b"}, // (C) + {0x1E6C, "\u1e6d"}, // (C) + {0x1E6E, "\u1e6f"}, // (C) + {0x1E70, "\u1e71"}, // (C) + {0x1E72, "\u1e73"}, // (C) + {0x1E74, "\u1e75"}, // (C) + {0x1E76, "\u1e77"}, // (C) + {0x1E78, "\u1e79"}, // (C) + {0x1E7A, "\u1e7b"}, // (C) + {0x1E7C, "\u1e7d"}, // (C) + {0x1E7E, "\u1e7f"}, // (C) + {0x1E80, "\u1e81"}, // (C) + {0x1E82, "\u1e83"}, // (C) + {0x1E84, "\u1e85"}, // (C) + {0x1E86, "\u1e87"}, // (C) + {0x1E88, "\u1e89"}, // (C) + {0x1E8A, "\u1e8b"}, // (C) + {0x1E8C, "\u1e8d"}, // (C) + {0x1E8E, "\u1e8f"}, // (C) + {0x1E90, "\u1e91"}, // (C) + {0x1E92, "\u1e93"}, // (C) + {0x1E94, "\u1e95"}, // (C) + {0x1E96, "h\u0331"}, // (F) + {0x1E97, "t\u0308"}, // (F) + {0x1E98, "w\u030a"}, // (F) + {0x1E99, "y\u030a"}, // (F) + {0x1E9A, "a\u02be"}, // (F) + {0x1E9B, "\u1e61"}, // (C) + {0x1E9E, "ss"}, // (F) + {0x1EA0, "\u1ea1"}, // (C) + {0x1EA2, "\u1ea3"}, // (C) + {0x1EA4, "\u1ea5"}, // (C) + {0x1EA6, "\u1ea7"}, // (C) + {0x1EA8, "\u1ea9"}, // (C) + {0x1EAA, "\u1eab"}, // (C) + {0x1EAC, "\u1ead"}, // (C) + {0x1EAE, "\u1eaf"}, // (C) + {0x1EB0, "\u1eb1"}, // (C) + {0x1EB2, "\u1eb3"}, // (C) + {0x1EB4, "\u1eb5"}, // (C) + {0x1EB6, "\u1eb7"}, // (C) + {0x1EB8, "\u1eb9"}, // (C) + {0x1EBA, "\u1ebb"}, // (C) + {0x1EBC, "\u1ebd"}, // (C) + {0x1EBE, "\u1ebf"}, // (C) + {0x1EC0, "\u1ec1"}, // (C) + {0x1EC2, "\u1ec3"}, // (C) + {0x1EC4, "\u1ec5"}, // (C) + {0x1EC6, "\u1ec7"}, // (C) + {0x1EC8, "\u1ec9"}, // (C) + {0x1ECA, "\u1ecb"}, // (C) + {0x1ECC, "\u1ecd"}, // (C) + {0x1ECE, "\u1ecf"}, // (C) + {0x1ED0, "\u1ed1"}, // (C) + {0x1ED2, "\u1ed3"}, // (C) + {0x1ED4, "\u1ed5"}, // (C) + {0x1ED6, "\u1ed7"}, // (C) + {0x1ED8, "\u1ed9"}, // (C) + {0x1EDA, "\u1edb"}, // (C) + {0x1EDC, "\u1edd"}, // (C) + {0x1EDE, "\u1edf"}, // (C) + {0x1EE0, "\u1ee1"}, // (C) + {0x1EE2, "\u1ee3"}, // (C) + {0x1EE4, "\u1ee5"}, // (C) + {0x1EE6, "\u1ee7"}, // (C) + {0x1EE8, "\u1ee9"}, // (C) + {0x1EEA, "\u1eeb"}, // (C) + {0x1EEC, "\u1eed"}, // (C) + {0x1EEE, "\u1eef"}, // (C) + {0x1EF0, "\u1ef1"}, // (C) + {0x1EF2, "\u1ef3"}, // (C) + {0x1EF4, "\u1ef5"}, // (C) + {0x1EF6, "\u1ef7"}, // (C) + {0x1EF8, "\u1ef9"}, // (C) + {0x1EFA, "\u1efb"}, // (C) + {0x1EFC, "\u1efd"}, // (C) + {0x1EFE, "\u1eff"}, // (C) + {0x1F08, "\u1f00"}, // (C) + {0x1F09, "\u1f01"}, // (C) + {0x1F0A, "\u1f02"}, // (C) + {0x1F0B, "\u1f03"}, // (C) + {0x1F0C, "\u1f04"}, // (C) + {0x1F0D, "\u1f05"}, // (C) + {0x1F0E, "\u1f06"}, // (C) + {0x1F0F, "\u1f07"}, // (C) + {0x1F18, "\u1f10"}, // (C) + {0x1F19, "\u1f11"}, // (C) + {0x1F1A, "\u1f12"}, // (C) + {0x1F1B, "\u1f13"}, // (C) + {0x1F1C, "\u1f14"}, // (C) + {0x1F1D, "\u1f15"}, // (C) + {0x1F28, "\u1f20"}, // (C) + {0x1F29, "\u1f21"}, // (C) + {0x1F2A, "\u1f22"}, // (C) + {0x1F2B, "\u1f23"}, // (C) + {0x1F2C, "\u1f24"}, // (C) + {0x1F2D, "\u1f25"}, // (C) + {0x1F2E, "\u1f26"}, // (C) + {0x1F2F, "\u1f27"}, // (C) + {0x1F38, "\u1f30"}, // (C) + {0x1F39, "\u1f31"}, // (C) + {0x1F3A, "\u1f32"}, // (C) + {0x1F3B, "\u1f33"}, // (C) + {0x1F3C, "\u1f34"}, // (C) + {0x1F3D, "\u1f35"}, // (C) + {0x1F3E, "\u1f36"}, // (C) + {0x1F3F, "\u1f37"}, // (C) + {0x1F48, "\u1f40"}, // (C) + {0x1F49, "\u1f41"}, // (C) + {0x1F4A, "\u1f42"}, // (C) + {0x1F4B, "\u1f43"}, // (C) + {0x1F4C, "\u1f44"}, // (C) + {0x1F4D, "\u1f45"}, // (C) + {0x1F50, "\u03c5\u0313"}, // (F) + {0x1F52, "\u03c5\u0313\u0300"}, // (F) + {0x1F54, "\u03c5\u0313\u0301"}, // (F) + {0x1F56, "\u03c5\u0313\u0342"}, // (F) + {0x1F59, "\u1f51"}, // (C) + {0x1F5B, "\u1f53"}, // (C) + {0x1F5D, "\u1f55"}, // (C) + {0x1F5F, "\u1f57"}, // (C) + {0x1F68, "\u1f60"}, // (C) + {0x1F69, "\u1f61"}, // (C) + {0x1F6A, "\u1f62"}, // (C) + {0x1F6B, "\u1f63"}, // (C) + {0x1F6C, "\u1f64"}, // (C) + {0x1F6D, "\u1f65"}, // (C) + {0x1F6E, "\u1f66"}, // (C) + {0x1F6F, "\u1f67"}, // (C) + {0x1F80, "\u1f00\u03b9"}, // (F) + {0x1F81, "\u1f01\u03b9"}, // (F) + {0x1F82, "\u1f02\u03b9"}, // (F) + {0x1F83, "\u1f03\u03b9"}, // (F) + {0x1F84, "\u1f04\u03b9"}, // (F) + {0x1F85, "\u1f05\u03b9"}, // (F) + {0x1F86, "\u1f06\u03b9"}, // (F) + {0x1F87, "\u1f07\u03b9"}, // (F) + {0x1F88, "\u1f00\u03b9"}, // (F) + {0x1F89, "\u1f01\u03b9"}, // (F) + {0x1F8A, "\u1f02\u03b9"}, // (F) + {0x1F8B, "\u1f03\u03b9"}, // (F) + {0x1F8C, "\u1f04\u03b9"}, // (F) + {0x1F8D, "\u1f05\u03b9"}, // (F) + {0x1F8E, "\u1f06\u03b9"}, // (F) + {0x1F8F, "\u1f07\u03b9"}, // (F) + {0x1F90, "\u1f20\u03b9"}, // (F) + {0x1F91, "\u1f21\u03b9"}, // (F) + {0x1F92, "\u1f22\u03b9"}, // (F) + {0x1F93, "\u1f23\u03b9"}, // (F) + {0x1F94, "\u1f24\u03b9"}, // (F) + {0x1F95, "\u1f25\u03b9"}, // (F) + {0x1F96, "\u1f26\u03b9"}, // (F) + {0x1F97, "\u1f27\u03b9"}, // (F) + {0x1F98, "\u1f20\u03b9"}, // (F) + {0x1F99, "\u1f21\u03b9"}, // (F) + {0x1F9A, "\u1f22\u03b9"}, // (F) + {0x1F9B, "\u1f23\u03b9"}, // (F) + {0x1F9C, "\u1f24\u03b9"}, // (F) + {0x1F9D, "\u1f25\u03b9"}, // (F) + {0x1F9E, "\u1f26\u03b9"}, // (F) + {0x1F9F, "\u1f27\u03b9"}, // (F) + {0x1FA0, "\u1f60\u03b9"}, // (F) + {0x1FA1, "\u1f61\u03b9"}, // (F) + {0x1FA2, "\u1f62\u03b9"}, // (F) + {0x1FA3, "\u1f63\u03b9"}, // (F) + {0x1FA4, "\u1f64\u03b9"}, // (F) + {0x1FA5, "\u1f65\u03b9"}, // (F) + {0x1FA6, "\u1f66\u03b9"}, // (F) + {0x1FA7, "\u1f67\u03b9"}, // (F) + {0x1FA8, "\u1f60\u03b9"}, // (F) + {0x1FA9, "\u1f61\u03b9"}, // (F) + {0x1FAA, "\u1f62\u03b9"}, // (F) + {0x1FAB, "\u1f63\u03b9"}, // (F) + {0x1FAC, "\u1f64\u03b9"}, // (F) + {0x1FAD, "\u1f65\u03b9"}, // (F) + {0x1FAE, "\u1f66\u03b9"}, // (F) + {0x1FAF, "\u1f67\u03b9"}, // (F) + {0x1FB2, "\u1f70\u03b9"}, // (F) + {0x1FB3, "\u03b1\u03b9"}, // (F) + {0x1FB4, "\u03ac\u03b9"}, // (F) + {0x1FB6, "\u03b1\u0342"}, // (F) + {0x1FB7, "\u03b1\u0342\u03b9"}, // (F) + {0x1FB8, "\u1fb0"}, // (C) + {0x1FB9, "\u1fb1"}, // (C) + {0x1FBA, "\u1f70"}, // (C) + {0x1FBB, "\u1f71"}, // (C) + {0x1FBC, "\u03b1\u03b9"}, // (F) + {0x1FBE, "\u03b9"}, // (C) + {0x1FC2, "\u1f74\u03b9"}, // (F) + {0x1FC3, "\u03b7\u03b9"}, // (F) + {0x1FC4, "\u03ae\u03b9"}, // (F) + {0x1FC6, "\u03b7\u0342"}, // (F) + {0x1FC7, "\u03b7\u0342\u03b9"}, // (F) + {0x1FC8, "\u1f72"}, // (C) + {0x1FC9, "\u1f73"}, // (C) + {0x1FCA, "\u1f74"}, // (C) + {0x1FCB, "\u1f75"}, // (C) + {0x1FCC, "\u03b7\u03b9"}, // (F) + {0x1FD2, "\u03b9\u0308\u0300"}, // (F) + {0x1FD3, "\u03b9\u0308\u0301"}, // (F) + {0x1FD6, "\u03b9\u0342"}, // (F) + {0x1FD7, "\u03b9\u0308\u0342"}, // (F) + {0x1FD8, "\u1fd0"}, // (C) + {0x1FD9, "\u1fd1"}, // (C) + {0x1FDA, "\u1f76"}, // (C) + {0x1FDB, "\u1f77"}, // (C) + {0x1FE2, "\u03c5\u0308\u0300"}, // (F) + {0x1FE3, "\u03c5\u0308\u0301"}, // (F) + {0x1FE4, "\u03c1\u0313"}, // (F) + {0x1FE6, "\u03c5\u0342"}, // (F) + {0x1FE7, "\u03c5\u0308\u0342"}, // (F) + {0x1FE8, "\u1fe0"}, // (C) + {0x1FE9, "\u1fe1"}, // (C) + {0x1FEA, "\u1f7a"}, // (C) + {0x1FEB, "\u1f7b"}, // (C) + {0x1FEC, "\u1fe5"}, // (C) + {0x1FF2, "\u1f7c\u03b9"}, // (F) + {0x1FF3, "\u03c9\u03b9"}, // (F) + {0x1FF4, "\u03ce\u03b9"}, // (F) + {0x1FF6, "\u03c9\u0342"}, // (F) + {0x1FF7, "\u03c9\u0342\u03b9"}, // (F) + {0x1FF8, "\u1f78"}, // (C) + {0x1FF9, "\u1f79"}, // (C) + {0x1FFA, "\u1f7c"}, // (C) + {0x1FFB, "\u1f7d"}, // (C) + {0x1FFC, "\u03c9\u03b9"}, // (F) + {0x2126, "\u03c9"}, // (C) + {0x212A, "k"}, // (C) + {0x212B, "\u00e5"}, // (C) + {0x2132, "\u214e"}, // (C) + {0x2160, "\u2170"}, // (C) + {0x2161, "\u2171"}, // (C) + {0x2162, "\u2172"}, // (C) + {0x2163, "\u2173"}, // (C) + {0x2164, "\u2174"}, // (C) + {0x2165, "\u2175"}, // (C) + {0x2166, "\u2176"}, // (C) + {0x2167, "\u2177"}, // (C) + {0x2168, "\u2178"}, // (C) + {0x2169, "\u2179"}, // (C) + {0x216A, "\u217a"}, // (C) + {0x216B, "\u217b"}, // (C) + {0x216C, "\u217c"}, // (C) + {0x216D, "\u217d"}, // (C) + {0x216E, "\u217e"}, // (C) + {0x216F, "\u217f"}, // (C) + {0x2183, "\u2184"}, // (C) + {0x24B6, "\u24d0"}, // (C) + {0x24B7, "\u24d1"}, // (C) + {0x24B8, "\u24d2"}, // (C) + {0x24B9, "\u24d3"}, // (C) + {0x24BA, "\u24d4"}, // (C) + {0x24BB, "\u24d5"}, // (C) + {0x24BC, "\u24d6"}, // (C) + {0x24BD, "\u24d7"}, // (C) + {0x24BE, "\u24d8"}, // (C) + {0x24BF, "\u24d9"}, // (C) + {0x24C0, "\u24da"}, // (C) + {0x24C1, "\u24db"}, // (C) + {0x24C2, "\u24dc"}, // (C) + {0x24C3, "\u24dd"}, // (C) + {0x24C4, "\u24de"}, // (C) + {0x24C5, "\u24df"}, // (C) + {0x24C6, "\u24e0"}, // (C) + {0x24C7, "\u24e1"}, // (C) + {0x24C8, "\u24e2"}, // (C) + {0x24C9, "\u24e3"}, // (C) + {0x24CA, "\u24e4"}, // (C) + {0x24CB, "\u24e5"}, // (C) + {0x24CC, "\u24e6"}, // (C) + {0x24CD, "\u24e7"}, // (C) + {0x24CE, "\u24e8"}, // (C) + {0x24CF, "\u24e9"}, // (C) + {0x2C00, "\u2c30"}, // (C) + {0x2C01, "\u2c31"}, // (C) + {0x2C02, "\u2c32"}, // (C) + {0x2C03, "\u2c33"}, // (C) + {0x2C04, "\u2c34"}, // (C) + {0x2C05, "\u2c35"}, // (C) + {0x2C06, "\u2c36"}, // (C) + {0x2C07, "\u2c37"}, // (C) + {0x2C08, "\u2c38"}, // (C) + {0x2C09, "\u2c39"}, // (C) + {0x2C0A, "\u2c3a"}, // (C) + {0x2C0B, "\u2c3b"}, // (C) + {0x2C0C, "\u2c3c"}, // (C) + {0x2C0D, "\u2c3d"}, // (C) + {0x2C0E, "\u2c3e"}, // (C) + {0x2C0F, "\u2c3f"}, // (C) + {0x2C10, "\u2c40"}, // (C) + {0x2C11, "\u2c41"}, // (C) + {0x2C12, "\u2c42"}, // (C) + {0x2C13, "\u2c43"}, // (C) + {0x2C14, "\u2c44"}, // (C) + {0x2C15, "\u2c45"}, // (C) + {0x2C16, "\u2c46"}, // (C) + {0x2C17, "\u2c47"}, // (C) + {0x2C18, "\u2c48"}, // (C) + {0x2C19, "\u2c49"}, // (C) + {0x2C1A, "\u2c4a"}, // (C) + {0x2C1B, "\u2c4b"}, // (C) + {0x2C1C, "\u2c4c"}, // (C) + {0x2C1D, "\u2c4d"}, // (C) + {0x2C1E, "\u2c4e"}, // (C) + {0x2C1F, "\u2c4f"}, // (C) + {0x2C20, "\u2c50"}, // (C) + {0x2C21, "\u2c51"}, // (C) + {0x2C22, "\u2c52"}, // (C) + {0x2C23, "\u2c53"}, // (C) + {0x2C24, "\u2c54"}, // (C) + {0x2C25, "\u2c55"}, // (C) + {0x2C26, "\u2c56"}, // (C) + {0x2C27, "\u2c57"}, // (C) + {0x2C28, "\u2c58"}, // (C) + {0x2C29, "\u2c59"}, // (C) + {0x2C2A, "\u2c5a"}, // (C) + {0x2C2B, "\u2c5b"}, // (C) + {0x2C2C, "\u2c5c"}, // (C) + {0x2C2D, "\u2c5d"}, // (C) + {0x2C2E, "\u2c5e"}, // (C) + {0x2C2F, "\u2c5f"}, // (C) + {0x2C60, "\u2c61"}, // (C) + {0x2C62, "\u026b"}, // (C) + {0x2C63, "\u1d7d"}, // (C) + {0x2C64, "\u027d"}, // (C) + {0x2C67, "\u2c68"}, // (C) + {0x2C69, "\u2c6a"}, // (C) + {0x2C6B, "\u2c6c"}, // (C) + {0x2C6D, "\u0251"}, // (C) + {0x2C6E, "\u0271"}, // (C) + {0x2C6F, "\u0250"}, // (C) + {0x2C70, "\u0252"}, // (C) + {0x2C72, "\u2c73"}, // (C) + {0x2C75, "\u2c76"}, // (C) + {0x2C7E, "\u023f"}, // (C) + {0x2C7F, "\u0240"}, // (C) + {0x2C80, "\u2c81"}, // (C) + {0x2C82, "\u2c83"}, // (C) + {0x2C84, "\u2c85"}, // (C) + {0x2C86, "\u2c87"}, // (C) + {0x2C88, "\u2c89"}, // (C) + {0x2C8A, "\u2c8b"}, // (C) + {0x2C8C, "\u2c8d"}, // (C) + {0x2C8E, "\u2c8f"}, // (C) + {0x2C90, "\u2c91"}, // (C) + {0x2C92, "\u2c93"}, // (C) + {0x2C94, "\u2c95"}, // (C) + {0x2C96, "\u2c97"}, // (C) + {0x2C98, "\u2c99"}, // (C) + {0x2C9A, "\u2c9b"}, // (C) + {0x2C9C, "\u2c9d"}, // (C) + {0x2C9E, "\u2c9f"}, // (C) + {0x2CA0, "\u2ca1"}, // (C) + {0x2CA2, "\u2ca3"}, // (C) + {0x2CA4, "\u2ca5"}, // (C) + {0x2CA6, "\u2ca7"}, // (C) + {0x2CA8, "\u2ca9"}, // (C) + {0x2CAA, "\u2cab"}, // (C) + {0x2CAC, "\u2cad"}, // (C) + {0x2CAE, "\u2caf"}, // (C) + {0x2CB0, "\u2cb1"}, // (C) + {0x2CB2, "\u2cb3"}, // (C) + {0x2CB4, "\u2cb5"}, // (C) + {0x2CB6, "\u2cb7"}, // (C) + {0x2CB8, "\u2cb9"}, // (C) + {0x2CBA, "\u2cbb"}, // (C) + {0x2CBC, "\u2cbd"}, // (C) + {0x2CBE, "\u2cbf"}, // (C) + {0x2CC0, "\u2cc1"}, // (C) + {0x2CC2, "\u2cc3"}, // (C) + {0x2CC4, "\u2cc5"}, // (C) + {0x2CC6, "\u2cc7"}, // (C) + {0x2CC8, "\u2cc9"}, // (C) + {0x2CCA, "\u2ccb"}, // (C) + {0x2CCC, "\u2ccd"}, // (C) + {0x2CCE, "\u2ccf"}, // (C) + {0x2CD0, "\u2cd1"}, // (C) + {0x2CD2, "\u2cd3"}, // (C) + {0x2CD4, "\u2cd5"}, // (C) + {0x2CD6, "\u2cd7"}, // (C) + {0x2CD8, "\u2cd9"}, // (C) + {0x2CDA, "\u2cdb"}, // (C) + {0x2CDC, "\u2cdd"}, // (C) + {0x2CDE, "\u2cdf"}, // (C) + {0x2CE0, "\u2ce1"}, // (C) + {0x2CE2, "\u2ce3"}, // (C) + {0x2CEB, "\u2cec"}, // (C) + {0x2CED, "\u2cee"}, // (C) + {0x2CF2, "\u2cf3"}, // (C) + {0xA640, "\ua641"}, // (C) + {0xA642, "\ua643"}, // (C) + {0xA644, "\ua645"}, // (C) + {0xA646, "\ua647"}, // (C) + {0xA648, "\ua649"}, // (C) + {0xA64A, "\ua64b"}, // (C) + {0xA64C, "\ua64d"}, // (C) + {0xA64E, "\ua64f"}, // (C) + {0xA650, "\ua651"}, // (C) + {0xA652, "\ua653"}, // (C) + {0xA654, "\ua655"}, // (C) + {0xA656, "\ua657"}, // (C) + {0xA658, "\ua659"}, // (C) + {0xA65A, "\ua65b"}, // (C) + {0xA65C, "\ua65d"}, // (C) + {0xA65E, "\ua65f"}, // (C) + {0xA660, "\ua661"}, // (C) + {0xA662, "\ua663"}, // (C) + {0xA664, "\ua665"}, // (C) + {0xA666, "\ua667"}, // (C) + {0xA668, "\ua669"}, // (C) + {0xA66A, "\ua66b"}, // (C) + {0xA66C, "\ua66d"}, // (C) + {0xA680, "\ua681"}, // (C) + {0xA682, "\ua683"}, // (C) + {0xA684, "\ua685"}, // (C) + {0xA686, "\ua687"}, // (C) + {0xA688, "\ua689"}, // (C) + {0xA68A, "\ua68b"}, // (C) + {0xA68C, "\ua68d"}, // (C) + {0xA68E, "\ua68f"}, // (C) + {0xA690, "\ua691"}, // (C) + {0xA692, "\ua693"}, // (C) + {0xA694, "\ua695"}, // (C) + {0xA696, "\ua697"}, // (C) + {0xA698, "\ua699"}, // (C) + {0xA69A, "\ua69b"}, // (C) + {0xA722, "\ua723"}, // (C) + {0xA724, "\ua725"}, // (C) + {0xA726, "\ua727"}, // (C) + {0xA728, "\ua729"}, // (C) + {0xA72A, "\ua72b"}, // (C) + {0xA72C, "\ua72d"}, // (C) + {0xA72E, "\ua72f"}, // (C) + {0xA732, "\ua733"}, // (C) + {0xA734, "\ua735"}, // (C) + {0xA736, "\ua737"}, // (C) + {0xA738, "\ua739"}, // (C) + {0xA73A, "\ua73b"}, // (C) + {0xA73C, "\ua73d"}, // (C) + {0xA73E, "\ua73f"}, // (C) + {0xA740, "\ua741"}, // (C) + {0xA742, "\ua743"}, // (C) + {0xA744, "\ua745"}, // (C) + {0xA746, "\ua747"}, // (C) + {0xA748, "\ua749"}, // (C) + {0xA74A, "\ua74b"}, // (C) + {0xA74C, "\ua74d"}, // (C) + {0xA74E, "\ua74f"}, // (C) + {0xA750, "\ua751"}, // (C) + {0xA752, "\ua753"}, // (C) + {0xA754, "\ua755"}, // (C) + {0xA756, "\ua757"}, // (C) + {0xA758, "\ua759"}, // (C) + {0xA75A, "\ua75b"}, // (C) + {0xA75C, "\ua75d"}, // (C) + {0xA75E, "\ua75f"}, // (C) + {0xA760, "\ua761"}, // (C) + {0xA762, "\ua763"}, // (C) + {0xA764, "\ua765"}, // (C) + {0xA766, "\ua767"}, // (C) + {0xA768, "\ua769"}, // (C) + {0xA76A, "\ua76b"}, // (C) + {0xA76C, "\ua76d"}, // (C) + {0xA76E, "\ua76f"}, // (C) + {0xA779, "\ua77a"}, // (C) + {0xA77B, "\ua77c"}, // (C) + {0xA77D, "\u1d79"}, // (C) + {0xA77E, "\ua77f"}, // (C) + {0xA780, "\ua781"}, // (C) + {0xA782, "\ua783"}, // (C) + {0xA784, "\ua785"}, // (C) + {0xA786, "\ua787"}, // (C) + {0xA78B, "\ua78c"}, // (C) + {0xA78D, "\u0265"}, // (C) + {0xA790, "\ua791"}, // (C) + {0xA792, "\ua793"}, // (C) + {0xA796, "\ua797"}, // (C) + {0xA798, "\ua799"}, // (C) + {0xA79A, "\ua79b"}, // (C) + {0xA79C, "\ua79d"}, // (C) + {0xA79E, "\ua79f"}, // (C) + {0xA7A0, "\ua7a1"}, // (C) + {0xA7A2, "\ua7a3"}, // (C) + {0xA7A4, "\ua7a5"}, // (C) + {0xA7A6, "\ua7a7"}, // (C) + {0xA7A8, "\ua7a9"}, // (C) + {0xA7AA, "\u0266"}, // (C) + {0xA7AB, "\u025c"}, // (C) + {0xA7AC, "\u0261"}, // (C) + {0xA7AD, "\u026c"}, // (C) + {0xA7AE, "\u026a"}, // (C) + {0xA7B0, "\u029e"}, // (C) + {0xA7B1, "\u0287"}, // (C) + {0xA7B2, "\u029d"}, // (C) + {0xA7B3, "\uab53"}, // (C) + {0xA7B4, "\ua7b5"}, // (C) + {0xA7B6, "\ua7b7"}, // (C) + {0xA7B8, "\ua7b9"}, // (C) + {0xA7BA, "\ua7bb"}, // (C) + {0xA7BC, "\ua7bd"}, // (C) + {0xA7BE, "\ua7bf"}, // (C) + {0xA7C0, "\ua7c1"}, // (C) + {0xA7C2, "\ua7c3"}, // (C) + {0xA7C4, "\ua794"}, // (C) + {0xA7C5, "\u0282"}, // (C) + {0xA7C6, "\u1d8e"}, // (C) + {0xA7C7, "\ua7c8"}, // (C) + {0xA7C9, "\ua7ca"}, // (C) + {0xA7CB, "\u0264"}, // (C) + {0xA7CC, "\ua7cd"}, // (C) + {0xA7CE, "\ua7cf"}, // (C) + {0xA7D0, "\ua7d1"}, // (C) + {0xA7D2, "\ua7d3"}, // (C) + {0xA7D4, "\ua7d5"}, // (C) + {0xA7D6, "\ua7d7"}, // (C) + {0xA7D8, "\ua7d9"}, // (C) + {0xA7DA, "\ua7db"}, // (C) + {0xA7DC, "\u019b"}, // (C) + {0xA7F5, "\ua7f6"}, // (C) + {0xAB70, "\u13a0"}, // (C) + {0xAB71, "\u13a1"}, // (C) + {0xAB72, "\u13a2"}, // (C) + {0xAB73, "\u13a3"}, // (C) + {0xAB74, "\u13a4"}, // (C) + {0xAB75, "\u13a5"}, // (C) + {0xAB76, "\u13a6"}, // (C) + {0xAB77, "\u13a7"}, // (C) + {0xAB78, "\u13a8"}, // (C) + {0xAB79, "\u13a9"}, // (C) + {0xAB7A, "\u13aa"}, // (C) + {0xAB7B, "\u13ab"}, // (C) + {0xAB7C, "\u13ac"}, // (C) + {0xAB7D, "\u13ad"}, // (C) + {0xAB7E, "\u13ae"}, // (C) + {0xAB7F, "\u13af"}, // (C) + {0xAB80, "\u13b0"}, // (C) + {0xAB81, "\u13b1"}, // (C) + {0xAB82, "\u13b2"}, // (C) + {0xAB83, "\u13b3"}, // (C) + {0xAB84, "\u13b4"}, // (C) + {0xAB85, "\u13b5"}, // (C) + {0xAB86, "\u13b6"}, // (C) + {0xAB87, "\u13b7"}, // (C) + {0xAB88, "\u13b8"}, // (C) + {0xAB89, "\u13b9"}, // (C) + {0xAB8A, "\u13ba"}, // (C) + {0xAB8B, "\u13bb"}, // (C) + {0xAB8C, "\u13bc"}, // (C) + {0xAB8D, "\u13bd"}, // (C) + {0xAB8E, "\u13be"}, // (C) + {0xAB8F, "\u13bf"}, // (C) + {0xAB90, "\u13c0"}, // (C) + {0xAB91, "\u13c1"}, // (C) + {0xAB92, "\u13c2"}, // (C) + {0xAB93, "\u13c3"}, // (C) + {0xAB94, "\u13c4"}, // (C) + {0xAB95, "\u13c5"}, // (C) + {0xAB96, "\u13c6"}, // (C) + {0xAB97, "\u13c7"}, // (C) + {0xAB98, "\u13c8"}, // (C) + {0xAB99, "\u13c9"}, // (C) + {0xAB9A, "\u13ca"}, // (C) + {0xAB9B, "\u13cb"}, // (C) + {0xAB9C, "\u13cc"}, // (C) + {0xAB9D, "\u13cd"}, // (C) + {0xAB9E, "\u13ce"}, // (C) + {0xAB9F, "\u13cf"}, // (C) + {0xABA0, "\u13d0"}, // (C) + {0xABA1, "\u13d1"}, // (C) + {0xABA2, "\u13d2"}, // (C) + {0xABA3, "\u13d3"}, // (C) + {0xABA4, "\u13d4"}, // (C) + {0xABA5, "\u13d5"}, // (C) + {0xABA6, "\u13d6"}, // (C) + {0xABA7, "\u13d7"}, // (C) + {0xABA8, "\u13d8"}, // (C) + {0xABA9, "\u13d9"}, // (C) + {0xABAA, "\u13da"}, // (C) + {0xABAB, "\u13db"}, // (C) + {0xABAC, "\u13dc"}, // (C) + {0xABAD, "\u13dd"}, // (C) + {0xABAE, "\u13de"}, // (C) + {0xABAF, "\u13df"}, // (C) + {0xABB0, "\u13e0"}, // (C) + {0xABB1, "\u13e1"}, // (C) + {0xABB2, "\u13e2"}, // (C) + {0xABB3, "\u13e3"}, // (C) + {0xABB4, "\u13e4"}, // (C) + {0xABB5, "\u13e5"}, // (C) + {0xABB6, "\u13e6"}, // (C) + {0xABB7, "\u13e7"}, // (C) + {0xABB8, "\u13e8"}, // (C) + {0xABB9, "\u13e9"}, // (C) + {0xABBA, "\u13ea"}, // (C) + {0xABBB, "\u13eb"}, // (C) + {0xABBC, "\u13ec"}, // (C) + {0xABBD, "\u13ed"}, // (C) + {0xABBE, "\u13ee"}, // (C) + {0xABBF, "\u13ef"}, // (C) + {0xFB00, "ff"}, // (F) + {0xFB01, "fi"}, // (F) + {0xFB02, "fl"}, // (F) + {0xFB03, "ffi"}, // (F) + {0xFB04, "ffl"}, // (F) + {0xFB05, "st"}, // (F) + {0xFB06, "st"}, // (F) + {0xFB13, "\u0574\u0576"}, // (F) + {0xFB14, "\u0574\u0565"}, // (F) + {0xFB15, "\u0574\u056b"}, // (F) + {0xFB16, "\u057e\u0576"}, // (F) + {0xFB17, "\u0574\u056d"}, // (F) + {0xFF21, "\uff41"}, // (C) + {0xFF22, "\uff42"}, // (C) + {0xFF23, "\uff43"}, // (C) + {0xFF24, "\uff44"}, // (C) + {0xFF25, "\uff45"}, // (C) + {0xFF26, "\uff46"}, // (C) + {0xFF27, "\uff47"}, // (C) + {0xFF28, "\uff48"}, // (C) + {0xFF29, "\uff49"}, // (C) + {0xFF2A, "\uff4a"}, // (C) + {0xFF2B, "\uff4b"}, // (C) + {0xFF2C, "\uff4c"}, // (C) + {0xFF2D, "\uff4d"}, // (C) + {0xFF2E, "\uff4e"}, // (C) + {0xFF2F, "\uff4f"}, // (C) + {0xFF30, "\uff50"}, // (C) + {0xFF31, "\uff51"}, // (C) + {0xFF32, "\uff52"}, // (C) + {0xFF33, "\uff53"}, // (C) + {0xFF34, "\uff54"}, // (C) + {0xFF35, "\uff55"}, // (C) + {0xFF36, "\uff56"}, // (C) + {0xFF37, "\uff57"}, // (C) + {0xFF38, "\uff58"}, // (C) + {0xFF39, "\uff59"}, // (C) + {0xFF3A, "\uff5a"}, // (C) + {0x10400, "\U00010428"}, // (C) + {0x10401, "\U00010429"}, // (C) + {0x10402, "\U0001042a"}, // (C) + {0x10403, "\U0001042b"}, // (C) + {0x10404, "\U0001042c"}, // (C) + {0x10405, "\U0001042d"}, // (C) + {0x10406, "\U0001042e"}, // (C) + {0x10407, "\U0001042f"}, // (C) + {0x10408, "\U00010430"}, // (C) + {0x10409, "\U00010431"}, // (C) + {0x1040A, "\U00010432"}, // (C) + {0x1040B, "\U00010433"}, // (C) + {0x1040C, "\U00010434"}, // (C) + {0x1040D, "\U00010435"}, // (C) + {0x1040E, "\U00010436"}, // (C) + {0x1040F, "\U00010437"}, // (C) + {0x10410, "\U00010438"}, // (C) + {0x10411, "\U00010439"}, // (C) + {0x10412, "\U0001043a"}, // (C) + {0x10413, "\U0001043b"}, // (C) + {0x10414, "\U0001043c"}, // (C) + {0x10415, "\U0001043d"}, // (C) + {0x10416, "\U0001043e"}, // (C) + {0x10417, "\U0001043f"}, // (C) + {0x10418, "\U00010440"}, // (C) + {0x10419, "\U00010441"}, // (C) + {0x1041A, "\U00010442"}, // (C) + {0x1041B, "\U00010443"}, // (C) + {0x1041C, "\U00010444"}, // (C) + {0x1041D, "\U00010445"}, // (C) + {0x1041E, "\U00010446"}, // (C) + {0x1041F, "\U00010447"}, // (C) + {0x10420, "\U00010448"}, // (C) + {0x10421, "\U00010449"}, // (C) + {0x10422, "\U0001044a"}, // (C) + {0x10423, "\U0001044b"}, // (C) + {0x10424, "\U0001044c"}, // (C) + {0x10425, "\U0001044d"}, // (C) + {0x10426, "\U0001044e"}, // (C) + {0x10427, "\U0001044f"}, // (C) + {0x104B0, "\U000104d8"}, // (C) + {0x104B1, "\U000104d9"}, // (C) + {0x104B2, "\U000104da"}, // (C) + {0x104B3, "\U000104db"}, // (C) + {0x104B4, "\U000104dc"}, // (C) + {0x104B5, "\U000104dd"}, // (C) + {0x104B6, "\U000104de"}, // (C) + {0x104B7, "\U000104df"}, // (C) + {0x104B8, "\U000104e0"}, // (C) + {0x104B9, "\U000104e1"}, // (C) + {0x104BA, "\U000104e2"}, // (C) + {0x104BB, "\U000104e3"}, // (C) + {0x104BC, "\U000104e4"}, // (C) + {0x104BD, "\U000104e5"}, // (C) + {0x104BE, "\U000104e6"}, // (C) + {0x104BF, "\U000104e7"}, // (C) + {0x104C0, "\U000104e8"}, // (C) + {0x104C1, "\U000104e9"}, // (C) + {0x104C2, "\U000104ea"}, // (C) + {0x104C3, "\U000104eb"}, // (C) + {0x104C4, "\U000104ec"}, // (C) + {0x104C5, "\U000104ed"}, // (C) + {0x104C6, "\U000104ee"}, // (C) + {0x104C7, "\U000104ef"}, // (C) + {0x104C8, "\U000104f0"}, // (C) + {0x104C9, "\U000104f1"}, // (C) + {0x104CA, "\U000104f2"}, // (C) + {0x104CB, "\U000104f3"}, // (C) + {0x104CC, "\U000104f4"}, // (C) + {0x104CD, "\U000104f5"}, // (C) + {0x104CE, "\U000104f6"}, // (C) + {0x104CF, "\U000104f7"}, // (C) + {0x104D0, "\U000104f8"}, // (C) + {0x104D1, "\U000104f9"}, // (C) + {0x104D2, "\U000104fa"}, // (C) + {0x104D3, "\U000104fb"}, // (C) + {0x10570, "\U00010597"}, // (C) + {0x10571, "\U00010598"}, // (C) + {0x10572, "\U00010599"}, // (C) + {0x10573, "\U0001059a"}, // (C) + {0x10574, "\U0001059b"}, // (C) + {0x10575, "\U0001059c"}, // (C) + {0x10576, "\U0001059d"}, // (C) + {0x10577, "\U0001059e"}, // (C) + {0x10578, "\U0001059f"}, // (C) + {0x10579, "\U000105a0"}, // (C) + {0x1057A, "\U000105a1"}, // (C) + {0x1057C, "\U000105a3"}, // (C) + {0x1057D, "\U000105a4"}, // (C) + {0x1057E, "\U000105a5"}, // (C) + {0x1057F, "\U000105a6"}, // (C) + {0x10580, "\U000105a7"}, // (C) + {0x10581, "\U000105a8"}, // (C) + {0x10582, "\U000105a9"}, // (C) + {0x10583, "\U000105aa"}, // (C) + {0x10584, "\U000105ab"}, // (C) + {0x10585, "\U000105ac"}, // (C) + {0x10586, "\U000105ad"}, // (C) + {0x10587, "\U000105ae"}, // (C) + {0x10588, "\U000105af"}, // (C) + {0x10589, "\U000105b0"}, // (C) + {0x1058A, "\U000105b1"}, // (C) + {0x1058C, "\U000105b3"}, // (C) + {0x1058D, "\U000105b4"}, // (C) + {0x1058E, "\U000105b5"}, // (C) + {0x1058F, "\U000105b6"}, // (C) + {0x10590, "\U000105b7"}, // (C) + {0x10591, "\U000105b8"}, // (C) + {0x10592, "\U000105b9"}, // (C) + {0x10594, "\U000105bb"}, // (C) + {0x10595, "\U000105bc"}, // (C) + {0x10C80, "\U00010cc0"}, // (C) + {0x10C81, "\U00010cc1"}, // (C) + {0x10C82, "\U00010cc2"}, // (C) + {0x10C83, "\U00010cc3"}, // (C) + {0x10C84, "\U00010cc4"}, // (C) + {0x10C85, "\U00010cc5"}, // (C) + {0x10C86, "\U00010cc6"}, // (C) + {0x10C87, "\U00010cc7"}, // (C) + {0x10C88, "\U00010cc8"}, // (C) + {0x10C89, "\U00010cc9"}, // (C) + {0x10C8A, "\U00010cca"}, // (C) + {0x10C8B, "\U00010ccb"}, // (C) + {0x10C8C, "\U00010ccc"}, // (C) + {0x10C8D, "\U00010ccd"}, // (C) + {0x10C8E, "\U00010cce"}, // (C) + {0x10C8F, "\U00010ccf"}, // (C) + {0x10C90, "\U00010cd0"}, // (C) + {0x10C91, "\U00010cd1"}, // (C) + {0x10C92, "\U00010cd2"}, // (C) + {0x10C93, "\U00010cd3"}, // (C) + {0x10C94, "\U00010cd4"}, // (C) + {0x10C95, "\U00010cd5"}, // (C) + {0x10C96, "\U00010cd6"}, // (C) + {0x10C97, "\U00010cd7"}, // (C) + {0x10C98, "\U00010cd8"}, // (C) + {0x10C99, "\U00010cd9"}, // (C) + {0x10C9A, "\U00010cda"}, // (C) + {0x10C9B, "\U00010cdb"}, // (C) + {0x10C9C, "\U00010cdc"}, // (C) + {0x10C9D, "\U00010cdd"}, // (C) + {0x10C9E, "\U00010cde"}, // (C) + {0x10C9F, "\U00010cdf"}, // (C) + {0x10CA0, "\U00010ce0"}, // (C) + {0x10CA1, "\U00010ce1"}, // (C) + {0x10CA2, "\U00010ce2"}, // (C) + {0x10CA3, "\U00010ce3"}, // (C) + {0x10CA4, "\U00010ce4"}, // (C) + {0x10CA5, "\U00010ce5"}, // (C) + {0x10CA6, "\U00010ce6"}, // (C) + {0x10CA7, "\U00010ce7"}, // (C) + {0x10CA8, "\U00010ce8"}, // (C) + {0x10CA9, "\U00010ce9"}, // (C) + {0x10CAA, "\U00010cea"}, // (C) + {0x10CAB, "\U00010ceb"}, // (C) + {0x10CAC, "\U00010cec"}, // (C) + {0x10CAD, "\U00010ced"}, // (C) + {0x10CAE, "\U00010cee"}, // (C) + {0x10CAF, "\U00010cef"}, // (C) + {0x10CB0, "\U00010cf0"}, // (C) + {0x10CB1, "\U00010cf1"}, // (C) + {0x10CB2, "\U00010cf2"}, // (C) + {0x10D50, "\U00010d70"}, // (C) + {0x10D51, "\U00010d71"}, // (C) + {0x10D52, "\U00010d72"}, // (C) + {0x10D53, "\U00010d73"}, // (C) + {0x10D54, "\U00010d74"}, // (C) + {0x10D55, "\U00010d75"}, // (C) + {0x10D56, "\U00010d76"}, // (C) + {0x10D57, "\U00010d77"}, // (C) + {0x10D58, "\U00010d78"}, // (C) + {0x10D59, "\U00010d79"}, // (C) + {0x10D5A, "\U00010d7a"}, // (C) + {0x10D5B, "\U00010d7b"}, // (C) + {0x10D5C, "\U00010d7c"}, // (C) + {0x10D5D, "\U00010d7d"}, // (C) + {0x10D5E, "\U00010d7e"}, // (C) + {0x10D5F, "\U00010d7f"}, // (C) + {0x10D60, "\U00010d80"}, // (C) + {0x10D61, "\U00010d81"}, // (C) + {0x10D62, "\U00010d82"}, // (C) + {0x10D63, "\U00010d83"}, // (C) + {0x10D64, "\U00010d84"}, // (C) + {0x10D65, "\U00010d85"}, // (C) + {0x118A0, "\U000118c0"}, // (C) + {0x118A1, "\U000118c1"}, // (C) + {0x118A2, "\U000118c2"}, // (C) + {0x118A3, "\U000118c3"}, // (C) + {0x118A4, "\U000118c4"}, // (C) + {0x118A5, "\U000118c5"}, // (C) + {0x118A6, "\U000118c6"}, // (C) + {0x118A7, "\U000118c7"}, // (C) + {0x118A8, "\U000118c8"}, // (C) + {0x118A9, "\U000118c9"}, // (C) + {0x118AA, "\U000118ca"}, // (C) + {0x118AB, "\U000118cb"}, // (C) + {0x118AC, "\U000118cc"}, // (C) + {0x118AD, "\U000118cd"}, // (C) + {0x118AE, "\U000118ce"}, // (C) + {0x118AF, "\U000118cf"}, // (C) + {0x118B0, "\U000118d0"}, // (C) + {0x118B1, "\U000118d1"}, // (C) + {0x118B2, "\U000118d2"}, // (C) + {0x118B3, "\U000118d3"}, // (C) + {0x118B4, "\U000118d4"}, // (C) + {0x118B5, "\U000118d5"}, // (C) + {0x118B6, "\U000118d6"}, // (C) + {0x118B7, "\U000118d7"}, // (C) + {0x118B8, "\U000118d8"}, // (C) + {0x118B9, "\U000118d9"}, // (C) + {0x118BA, "\U000118da"}, // (C) + {0x118BB, "\U000118db"}, // (C) + {0x118BC, "\U000118dc"}, // (C) + {0x118BD, "\U000118dd"}, // (C) + {0x118BE, "\U000118de"}, // (C) + {0x118BF, "\U000118df"}, // (C) + {0x16E40, "\U00016e60"}, // (C) + {0x16E41, "\U00016e61"}, // (C) + {0x16E42, "\U00016e62"}, // (C) + {0x16E43, "\U00016e63"}, // (C) + {0x16E44, "\U00016e64"}, // (C) + {0x16E45, "\U00016e65"}, // (C) + {0x16E46, "\U00016e66"}, // (C) + {0x16E47, "\U00016e67"}, // (C) + {0x16E48, "\U00016e68"}, // (C) + {0x16E49, "\U00016e69"}, // (C) + {0x16E4A, "\U00016e6a"}, // (C) + {0x16E4B, "\U00016e6b"}, // (C) + {0x16E4C, "\U00016e6c"}, // (C) + {0x16E4D, "\U00016e6d"}, // (C) + {0x16E4E, "\U00016e6e"}, // (C) + {0x16E4F, "\U00016e6f"}, // (C) + {0x16E50, "\U00016e70"}, // (C) + {0x16E51, "\U00016e71"}, // (C) + {0x16E52, "\U00016e72"}, // (C) + {0x16E53, "\U00016e73"}, // (C) + {0x16E54, "\U00016e74"}, // (C) + {0x16E55, "\U00016e75"}, // (C) + {0x16E56, "\U00016e76"}, // (C) + {0x16E57, "\U00016e77"}, // (C) + {0x16E58, "\U00016e78"}, // (C) + {0x16E59, "\U00016e79"}, // (C) + {0x16E5A, "\U00016e7a"}, // (C) + {0x16E5B, "\U00016e7b"}, // (C) + {0x16E5C, "\U00016e7c"}, // (C) + {0x16E5D, "\U00016e7d"}, // (C) + {0x16E5E, "\U00016e7e"}, // (C) + {0x16E5F, "\U00016e7f"}, // (C) + {0x16EA0, "\U00016ebb"}, // (C) + {0x16EA1, "\U00016ebc"}, // (C) + {0x16EA2, "\U00016ebd"}, // (C) + {0x16EA3, "\U00016ebe"}, // (C) + {0x16EA4, "\U00016ebf"}, // (C) + {0x16EA5, "\U00016ec0"}, // (C) + {0x16EA6, "\U00016ec1"}, // (C) + {0x16EA7, "\U00016ec2"}, // (C) + {0x16EA8, "\U00016ec3"}, // (C) + {0x16EA9, "\U00016ec4"}, // (C) + {0x16EAA, "\U00016ec5"}, // (C) + {0x16EAB, "\U00016ec6"}, // (C) + {0x16EAC, "\U00016ec7"}, // (C) + {0x16EAD, "\U00016ec8"}, // (C) + {0x16EAE, "\U00016ec9"}, // (C) + {0x16EAF, "\U00016eca"}, // (C) + {0x16EB0, "\U00016ecb"}, // (C) + {0x16EB1, "\U00016ecc"}, // (C) + {0x16EB2, "\U00016ecd"}, // (C) + {0x16EB3, "\U00016ece"}, // (C) + {0x16EB4, "\U00016ecf"}, // (C) + {0x16EB5, "\U00016ed0"}, // (C) + {0x16EB6, "\U00016ed1"}, // (C) + {0x16EB7, "\U00016ed2"}, // (C) + {0x16EB8, "\U00016ed3"}, // (C) + {0x1E900, "\U0001e922"}, // (C) + {0x1E901, "\U0001e923"}, // (C) + {0x1E902, "\U0001e924"}, // (C) + {0x1E903, "\U0001e925"}, // (C) + {0x1E904, "\U0001e926"}, // (C) + {0x1E905, "\U0001e927"}, // (C) + {0x1E906, "\U0001e928"}, // (C) + {0x1E907, "\U0001e929"}, // (C) + {0x1E908, "\U0001e92a"}, // (C) + {0x1E909, "\U0001e92b"}, // (C) + {0x1E90A, "\U0001e92c"}, // (C) + {0x1E90B, "\U0001e92d"}, // (C) + {0x1E90C, "\U0001e92e"}, // (C) + {0x1E90D, "\U0001e92f"}, // (C) + {0x1E90E, "\U0001e930"}, // (C) + {0x1E90F, "\U0001e931"}, // (C) + {0x1E910, "\U0001e932"}, // (C) + {0x1E911, "\U0001e933"}, // (C) + {0x1E912, "\U0001e934"}, // (C) + {0x1E913, "\U0001e935"}, // (C) + {0x1E914, "\U0001e936"}, // (C) + {0x1E915, "\U0001e937"}, // (C) + {0x1E916, "\U0001e938"}, // (C) + {0x1E917, "\U0001e939"}, // (C) + {0x1E918, "\U0001e93a"}, // (C) + {0x1E919, "\U0001e93b"}, // (C) + {0x1E91A, "\U0001e93c"}, // (C) + {0x1E91B, "\U0001e93d"}, // (C) + {0x1E91C, "\U0001e93e"}, // (C) + {0x1E91D, "\U0001e93f"}, // (C) + {0x1E91E, "\U0001e940"}, // (C) + {0x1E91F, "\U0001e941"}, // (C) + {0x1E920, "\U0001e942"}, // (C) + {0x1E921, "\U0001e943"}, // (C) +} diff --git a/uts39/example_test.go b/uts39/example_test.go index b69dfe2..fd5a082 100644 --- a/uts39/example_test.go +++ b/uts39/example_test.go @@ -41,7 +41,7 @@ func ExampleGetRestrictionLevel() { examples := []string{ "hello_world", // ASCII-Only "café", // Single-Script (Latin) - "hello世界", // Minimally-Restrictive (Latin + Han) + "hello世界", // Highly-Restrictive (Latin + Han ⊂ Latn + Jpan) "hello мир", // Minimally-Restrictive (Latin + Cyrillic) } @@ -53,7 +53,7 @@ func ExampleGetRestrictionLevel() { // Output: // "hello_world": ASCII-Only // "café": Single-Script - // "hello世界": Minimally-Restrictive + // "hello世界": Highly-Restrictive // "hello мир": Minimally-Restrictive } diff --git a/uts39/generate_casefold.go b/uts39/generate_casefold.go new file mode 100644 index 0000000..4fb3db0 --- /dev/null +++ b/uts39/generate_casefold.go @@ -0,0 +1,169 @@ +//go:build ignore +// +build ignore + +// This program generates casefold_data.go from Unicode CaseFolding.txt. +// Run with: go run generate_casefold.go +// +// UTS #39 specifies "full case folding" for the skeleton algorithm, which +// corresponds to including the status 'C' (common) and 'F' (full) mappings +// from CaseFolding.txt and skipping 'S' (simple) and 'T' (Turkic) mappings. +// +// See: https://www.unicode.org/reports/tr39/#Confusable_Detection +// https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt + +package main + +import ( + "bufio" + "fmt" + "io" + "net/http" + "os" + "sort" + "strconv" + "strings" +) + +const ( + caseFoldingURL = "https://www.unicode.org/Public/17.0.0/ucd/CaseFolding.txt" + unicodeVersion = "17.0.0" +) + +type caseFoldMapping struct { + source rune + target string // May be multiple runes (full folding) + status string // C or F +} + +func main() { + fmt.Println("Downloading CaseFolding.txt...") + resp, err := http.Get(caseFoldingURL) + if err != nil { + fmt.Fprintf(os.Stderr, "Error downloading CaseFolding.txt: %v\n", err) + os.Exit(1) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "Error: HTTP %d\n", resp.StatusCode) + os.Exit(1) + } + + mappings := parseCaseFolding(resp.Body) + fmt.Printf("Parsed %d C+F case fold mappings\n", len(mappings)) + + if err := generateCaseFoldFile(mappings); err != nil { + fmt.Fprintf(os.Stderr, "Error writing casefold_data.go: %v\n", err) + os.Exit(1) + } + fmt.Println("Generated casefold_data.go") +} + +func parseCaseFolding(r io.Reader) []caseFoldMapping { + scanner := bufio.NewScanner(r) + var mappings []caseFoldMapping + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + // Skip comments and empty lines. + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + // Strip inline comments. + if idx := strings.Index(line, "#"); idx >= 0 { + line = strings.TrimSpace(line[:idx]) + } + + // Format: ; ; ; + parts := strings.Split(line, ";") + if len(parts) < 3 { + continue + } + + sourceStr := strings.TrimSpace(parts[0]) + status := strings.TrimSpace(parts[1]) + targetStr := strings.TrimSpace(parts[2]) + + // UTS #39 calls for "full case folding": include C (common) and F (full) + // mappings; exclude S (simple) and T (Turkic). + if status != "C" && status != "F" { + continue + } + + sourceVal, err := strconv.ParseInt(sourceStr, 16, 32) + if err != nil { + continue + } + + targetRunes := parseCodePoints(targetStr) + if len(targetRunes) == 0 { + continue + } + + mappings = append(mappings, caseFoldMapping{ + source: rune(sourceVal), + target: string(targetRunes), + status: status, + }) + } + + // Sort by source code point for binary search. + sort.Slice(mappings, func(i, j int) bool { + return mappings[i].source < mappings[j].source + }) + + return mappings +} + +func parseCodePoints(s string) []rune { + parts := strings.Fields(s) + runes := make([]rune, 0, len(parts)) + + for _, part := range parts { + val, err := strconv.ParseInt(part, 16, 32) + if err != nil { + continue + } + runes = append(runes, rune(val)) + } + + return runes +} + +func generateCaseFoldFile(mappings []caseFoldMapping) error { + f, err := os.Create("casefold_data.go") + if err != nil { + return err + } + defer f.Close() + + fmt.Fprintf(f, "// Code generated by generate_casefold.go. DO NOT EDIT.\n") + fmt.Fprintf(f, "// Source: %s\n", caseFoldingURL) + fmt.Fprintf(f, "// Unicode version: %s\n", unicodeVersion) + fmt.Fprintf(f, "// Includes status C (common) and F (full) mappings only,\n") + fmt.Fprintf(f, "// per UTS #39's \"full case folding\" requirement.\n") + fmt.Fprintf(f, "\npackage uts39\n\n") + + fmt.Fprintf(f, "// caseFoldEntry represents a case folding mapping from a single\n") + fmt.Fprintf(f, "// source code point to (possibly multiple) target code points.\n") + fmt.Fprintf(f, "type caseFoldEntry struct {\n") + fmt.Fprintf(f, "\tsource rune\n") + fmt.Fprintf(f, "\ttarget string\n") + fmt.Fprintf(f, "}\n\n") + // The remaining output is run through gofmt by the caller; this generator + // emits canonical tab-indented Go source. + + fmt.Fprintf(f, "// caseFoldData contains the full case folding mappings (C + F)\n") + fmt.Fprintf(f, "// from CaseFolding.txt. Sorted by source for binary search.\n") + fmt.Fprintf(f, "var caseFoldData = []caseFoldEntry{\n") + + for _, m := range mappings { + fmt.Fprintf(f, "\t{0x%04X, %s}, // (%s)\n", + m.source, fmt.Sprintf("%+q", m.target), m.status) + } + + fmt.Fprintf(f, "}\n") + return nil +} diff --git a/uts39/official_test.go b/uts39/official_test.go index b499d30..46fc25e 100644 --- a/uts39/official_test.go +++ b/uts39/official_test.go @@ -182,7 +182,11 @@ func TestRestrictionLevelConformance(t *testing.T) { {"Latin", "café", SingleScript, SingleScript}, {"Cyrillic", "привет", SingleScript, SingleScript}, {"Han", "你好", SingleScript, SingleScript}, - {"Latin+Han", "hello世界", MinimallyRestrictive, MinimallyRestrictive}, + // Per UAX #39 §5.2 Table 1, Latin + Han is a subset of Latn + Jpan, + // so it qualifies as HighlyRestrictive. + {"Latin+Han", "hello世界", HighlyRestrictive, HighlyRestrictive}, + // Latin + Cyrillic is excluded from ModeratelyRestrictive, so it + // lands at MinimallyRestrictive. {"Latin+Cyrillic", "hello мир", MinimallyRestrictive, MinimallyRestrictive}, } diff --git a/uts39/uts39.go b/uts39/uts39.go index f4562be..e3f60be 100644 --- a/uts39/uts39.go +++ b/uts39/uts39.go @@ -14,7 +14,11 @@ // The skeleton algorithm identifies visually confusable strings by // normalizing them to a canonical form: // -// skeleton(X) = toNFD(toCaseFold(toNFKD(X))) +// skeleton(X) = applyConfusables( toNFD( toCaseFold( toNFKD(X) ) ) ) +// +// (The base transform follows the legacy UTS #39 ordering documented in this +// package; the confusables map is applied to the final NFD form and the +// result is iterated to a fixed point.) // // Two strings are confusable if their skeletons are identical. // @@ -56,7 +60,7 @@ package uts39 import ( - "strings" + "fmt" "github.com/SCKelemen/unicode/v6/uax24" "github.com/SCKelemen/unicode/v6/uax31" @@ -110,39 +114,63 @@ func (l RestrictionLevel) String() string { // Skeleton returns the skeleton of a string for confusable detection. // -// The skeleton algorithm normalizes strings to identify visual confusability: +// The skeleton algorithm normalizes strings to identify visual confusability. +// This implementation follows the legacy UTS #39 ordering documented for this +// package: +// +// skeleton(X) = applyConfusables( toNFD( toCaseFold( toNFKD(X) ) ) ) +// +// where: +// +// - toNFKD applies Unicode Normalization Form KD (compatibility decomposition). +// - toCaseFold applies "full" Unicode case folding (status C+F mappings from +// CaseFolding.txt); this is what causes "ß" to fold to "ss" and what +// distinguishes proper folding from a simple strings.ToLower call. +// - toNFD applies Unicode Normalization Form D (canonical decomposition). +// - applyConfusables applies the confusables.txt prototype mappings to the +// final NFD form. // -// skeleton(X) = toNFD(toCaseFold(toNFKD(X))) +// After the initial transform, the (NFD → applyConfusables) tail is repeated +// until a fixed point is reached. The function is mathematically idempotent +// per UTS #39, so convergence is expected within a handful of iterations. +// A defensive safety cap (16 iterations) panics on non-convergence to surface +// any future data regressions rather than silently truncate. // // Two strings are confusable if their skeletons are equal. // // Example: // -// skeleton("paypal") == skeleton("pаypal") // true (Cyrillic 'а') +// Skeleton("paypal") == Skeleton("pаypal") // true (Cyrillic 'а') +// Skeleton("ß") == Skeleton("ss") // true (full case fold) // // See: https://www.unicode.org/reports/tr39/#Confusable_Detection func Skeleton(s string) string { - // Step 1: Apply NFKD normalization and confusable mappings + // Base transform: NFKD → caseFold → NFD → applyConfusables. s = uts15.NFKD(s) + s = caseFold(s) + s = uts15.NFD(s) s = applyConfusables(s) - // Step 2: Apply case folding (convert to lowercase) - s = strings.ToLower(s) - - // Step 3: Apply NFD normalization and confusable mappings until fixed point - // Most strings reach fixed point in 1-2 iterations - for i := 0; i < 3; i++ { // Limit iterations to prevent infinite loops + // Iterate the (NFD → confusables) tail to a fixed point. Per UTS #39 + // the skeleton transform is idempotent, but a confusable replacement + // may produce a sequence that is not in NFD or that itself maps further, + // so a small number of iterations may be required. + const maxIterations = 16 + for i := 0; i < maxIterations; i++ { prev := s s = uts15.NFD(s) s = applyConfusables(s) - - // Stop if we've reached a fixed point if s == prev { - break + return s } } - return s + // Reaching this point means the skeleton transform did not converge + // within the safety cap, which would indicate a regression in the + // confusables data (e.g. a mapping cycle). Surface this loudly rather + // than silently truncate the result. + panic(fmt.Sprintf("uts39: Skeleton failed to converge after %d iterations (input=%q)", + maxIterations, s)) } // AreConfusable reports whether two strings are visually confusable. @@ -195,15 +223,93 @@ func getConfusableTarget(r rune) string { return "" } -// GetRestrictionLevel returns the restriction level of a string. +// caseFold applies Unicode "full" case folding to s, using the C+F mappings +// from CaseFolding.txt. This is what UTS #39 §4 calls toCaseFold. +// +// Unlike strings.ToLower, full case folding can produce a string longer than +// its input (e.g. "ß" → "ss", "ffi" → "ffi", "İ" → "i\u0307") and uses Unicode's +// language-neutral folding (no Turkic-specific 'T' mappings). +func caseFold(s string) string { + // Fast path: pure ASCII. Only A-Z need folding; everything else is + // already at its folded form. + allASCII := true + needsFold := false + for i := 0; i < len(s); i++ { + b := s[i] + if b >= 0x80 { + allASCII = false + break + } + if b >= 'A' && b <= 'Z' { + needsFold = true + } + } + if allASCII { + if !needsFold { + return s + } + // ASCII-only with at least one upper-case letter: fold in place. + buf := make([]byte, len(s)) + for i := 0; i < len(s); i++ { + b := s[i] + if b >= 'A' && b <= 'Z' { + b += 'a' - 'A' + } + buf[i] = b + } + return string(buf) + } + + // General path: walk runes and substitute via the case fold table. + // Pre-grow capacity slightly to absorb the common 1→1 case without + // reallocation, while still allowing 1→N expansion (e.g. ß → ss). + var b []byte + if cap(b) < len(s) { + b = make([]byte, 0, len(s)+8) + } + for _, r := range s { + if folded := getCaseFoldTarget(r); folded != "" { + b = append(b, folded...) + } else { + b = append(b, string(r)...) + } + } + return string(b) +} + +// getCaseFoldTarget returns the case-folded target for a rune, or empty +// string if r folds to itself. Uses binary search over caseFoldData. +func getCaseFoldTarget(r rune) string { + // caseFoldData entries are sorted by source code point. + lo, hi := 0, len(caseFoldData) + for lo < hi { + mid := lo + (hi-lo)/2 + if r < caseFoldData[mid].source { + hi = mid + } else if r > caseFoldData[mid].source { + lo = mid + 1 + } else { + return caseFoldData[mid].target + } + } + return "" +} + +// GetRestrictionLevel returns the restriction level of a string per +// UAX #39 §5.2 Table 1. +// +// Restriction levels are ordered from most to least restrictive: // -// Restriction levels from most to least restrictive: -// - ASCIIOnly: Only ASCII characters -// - SingleScript: One script (excluding Common/Inherited) -// - HighlyRestrictive: One script + Common + Inherited -// - ModeratelyRestrictive: Multiple scripts following specific rules -// - MinimallyRestrictive: Latin + one other script -// - Unrestricted: Any character combination +// - ASCIIOnly: Only ASCII characters. +// - SingleScript: Exactly one script (Common/Inherited ignored). +// - HighlyRestrictive: Single script, OR Latin combined with one of the +// JCB script groups: {Han, Hiragana, Katakana} (Latn+Jpan), +// {Han, Bopomofo} (Latn+Hanb), or {Han, Hangul} (Latn+Kore). +// - ModeratelyRestrictive: Latin plus exactly one other script that is not +// Cyrillic or Greek (per UAX #39 §5.2 Table 1). +// - MinimallyRestrictive: Any other multi-script combination (e.g. Latin + +// Cyrillic, Latin + Greek). +// - Unrestricted: Empty string / fallback. // // See: https://www.unicode.org/reports/tr39/#Restriction_Level_Detection func GetRestrictionLevel(s string) RestrictionLevel { @@ -211,7 +317,7 @@ func GetRestrictionLevel(s string) RestrictionLevel { return Unrestricted } - // Check if ASCII-only + // Check if ASCII-only. isASCII := true for _, r := range s { if r > 127 { @@ -223,10 +329,8 @@ func GetRestrictionLevel(s string) RestrictionLevel { return ASCIIOnly } - // Get all scripts used + // Collect all scripts and filter out Common / Inherited. scripts := GetIdentifierScripts(s) - - // Filter out Common and Inherited mainScripts := make([]uax24.Script, 0, len(scripts)) for _, script := range scripts { if script != uax24.ScriptCommon && script != uax24.ScriptInherited { @@ -234,39 +338,95 @@ func GetRestrictionLevel(s string) RestrictionLevel { } } - // SingleScript: only one main script + // SingleScript: only one resolved script. if len(mainScripts) == 1 { return SingleScript } - // HighlyRestrictive: one script + Common + Inherited is same as SingleScript - // for our purposes since we filtered out Common/Inherited - if len(mainScripts) == 1 { + // HighlyRestrictive: per UAX #39 §5.2 Table 1, the set of scripts (after + // removing Common and Inherited) must be a subset of one of the following + // CJK augmentations of Latin: + // + // a) {Latin, Han, Hiragana, Katakana} (Latn + Jpan) + // b) {Latin, Han, Bopomofo} (Latn + Hanb) + // c) {Latin, Han, Hangul} (Latn + Kore) + // + // These are intentionally checked as subset relations so that, e.g., + // "Latin + Han" alone also qualifies (it is a subset of all three). + if isScriptSubset(mainScripts, latnJpanSet) || + isScriptSubset(mainScripts, latnHanbSet) || + isScriptSubset(mainScripts, latnKoreSet) { return HighlyRestrictive } - // Check if Latin is present - hasLatin := false - for _, script := range mainScripts { - if script == uax24.ScriptLatin { - hasLatin = true - break + // ModeratelyRestrictive: Latin + exactly one other script, excluding the + // historically high-confusability scripts Cyrillic and Greek per + // UAX #39 §5.2 Table 1. + hasLatin := containsScript(mainScripts, uax24.ScriptLatin) + if hasLatin && len(mainScripts) == 2 { + other := otherScript(mainScripts, uax24.ScriptLatin) + if other != uax24.ScriptCyrillic && other != uax24.ScriptGreek { + return ModeratelyRestrictive } } - // MinimallyRestrictive: Latin + exactly one other script - if hasLatin && len(mainScripts) == 2 { - return MinimallyRestrictive + // MinimallyRestrictive: any remaining multi-script combination + // (e.g. Latin + Cyrillic, Latin + Greek, or three or more scripts not + // captured by HighlyRestrictive). + return MinimallyRestrictive +} + +// latnJpanSet is the script set for Latin + Japanese (Latn + Jpan). +var latnJpanSet = map[uax24.Script]struct{}{ + uax24.ScriptLatin: {}, + uax24.ScriptHan: {}, + uax24.ScriptHiragana: {}, + uax24.ScriptKatakana: {}, +} + +// latnHanbSet is the script set for Latin + Han-with-Bopomofo (Latn + Hanb). +var latnHanbSet = map[uax24.Script]struct{}{ + uax24.ScriptLatin: {}, + uax24.ScriptHan: {}, + uax24.ScriptBopomofo: {}, +} + +// latnKoreSet is the script set for Latin + Korean (Latn + Kore). +var latnKoreSet = map[uax24.Script]struct{}{ + uax24.ScriptLatin: {}, + uax24.ScriptHan: {}, + uax24.ScriptHangul: {}, +} + +// isScriptSubset reports whether every script in scripts is also in allowed. +func isScriptSubset(scripts []uax24.Script, allowed map[uax24.Script]struct{}) bool { + for _, s := range scripts { + if _, ok := allowed[s]; !ok { + return false + } } + return true +} - // ModeratelyRestrictive: multiple scripts with specific allowed combinations - // For simplicity, we consider any multi-script as moderately restrictive - // unless it matches minimally restrictive - if len(mainScripts) > 1 { - return ModeratelyRestrictive +// containsScript reports whether scripts contains target. +func containsScript(scripts []uax24.Script, target uax24.Script) bool { + for _, s := range scripts { + if s == target { + return true + } } + return false +} - return Unrestricted +// otherScript returns the first script in scripts that is not equal to skip. +// Caller must ensure such a script exists. +func otherScript(scripts []uax24.Script, skip uax24.Script) uax24.Script { + for _, s := range scripts { + if s != skip { + return s + } + } + return uax24.ScriptUnknown } // GetIdentifierScripts returns the scripts used in an identifier string. diff --git a/uts39/uts39_test.go b/uts39/uts39_test.go index 104ab59..d58ec18 100644 --- a/uts39/uts39_test.go +++ b/uts39/uts39_test.go @@ -106,15 +106,65 @@ func TestGetRestrictionLevel(t *testing.T) { want: SingleScript, }, { - name: "Latin + Han (minimally restrictive)", + // Latin + Han is a subset of {Latin, Han, Hiragana, Katakana} + // (Latn + Jpan), so this is HighlyRestrictive per UAX #39 §5.2 + // Table 1. + name: "Latin + Han (highly restrictive)", input: "hello世界", - want: MinimallyRestrictive, + want: HighlyRestrictive, }, { + // Latin + Cyrillic falls out of ModeratelyRestrictive because + // Cyrillic is explicitly excluded; it lands at MinimallyRestrictive. name: "Mixed Cyrillic and Latin", input: "hello мир", want: MinimallyRestrictive, }, + { + // Latin + Han + Hiragana matches Latn + Jpan exactly. + name: "Latin + Han + Hiragana (highly restrictive)", + input: "hello漢字ひらがな", + want: HighlyRestrictive, + }, + { + // Latin + Han + Katakana also fits Latn + Jpan. + name: "Latin + Han + Katakana (highly restrictive)", + input: "helloカタカナ漢", + want: HighlyRestrictive, + }, + { + // Latin + Han + Hangul matches Latn + Kore. + name: "Latin + Han + Hangul (highly restrictive)", + input: "hello漢字ㅎ", + want: HighlyRestrictive, + }, + { + // Latin + Han + Bopomofo matches Latn + Hanb. + name: "Latin + Han + Bopomofo (highly restrictive)", + input: "hello漢ㄅ", + want: HighlyRestrictive, + }, + { + // Latin + Arabic is ModeratelyRestrictive: a single non-excluded + // Recommended script paired with Latin. + name: "Latin + Arabic (moderately restrictive)", + input: "helloمرحبا", + want: ModeratelyRestrictive, + }, + { + // Latin + Greek collapses to MinimallyRestrictive because Greek + // is excluded from the ModeratelyRestrictive set. + name: "Latin + Greek (minimally restrictive)", + input: "helloαβ", + want: MinimallyRestrictive, + }, + { + // Latin + Cyrillic + Greek is too broad for any of the higher + // levels, so it lands at MinimallyRestrictive. + name: "Latin + Cyrillic + Greek (minimally restrictive)", + input: "helloмαβ", + want: MinimallyRestrictive, + }, { name: "Empty string", input: "",