patch 8.0.1203: terminal window mistreats composing characters

Problem:    Terminal window mistreats composing characters.
Solution:   Count composing characters with the base character. (Ozaki Kiichi,
            closes #2195)

Author: brammool

src/mbyte.c

@@ -1402,6 +1402,8 @@ static struct interval ambiguous[] =
     int
 utf_uint2cells(UINT32_T c)
 {
+    if (c >= 0x100 && utf_iscomposing((int)c))
+	return 0;
     return utf_char2cells((int)c);
 }
 #endif

src/terminal.c

@@ -40,12 +40,9 @@
  * TODO:
  * - in GUI vertical split causes problems.  Cursor is flickering. (Hirohito
  *   Higashi, 2017 Sep 19)
- * - patch to handle composing characters. (Ozaki Kiichi, #2195)
  * - double click in Window toolbar starts Visual mode (but not always?).
  * - Shift-Tab does not work.
  * - after resizing windows overlap. (Boris Staletic, #2164)
- * - :wall gives an error message. (Marius Gedminas, #2190)
- *   patch suggested by Yasuhiro Matsumoto, Oct 10
  * - Redirecting output does not work on MS-Windows, Test_terminal_redir_file()
  *   is disabled.
  * - cursor blinks in terminal on widows with a timer. (xtal8, #2142)
@@ -2299,7 +2296,6 @@ term_update_window(win_T *wp)
 		if (vterm_screen_get_cell(screen, pos, &cell) == 0)
 		    vim_memset(&cell, 0, sizeof(cell));
 
-		/* TODO: composing chars */
 		c = cell.chars[0];
 		if (c == NUL)
 		{
@@ -2311,7 +2307,18 @@ term_update_window(win_T *wp)
 		{
 		    if (enc_utf8)
 		    {
-			if (c >= 0x80)
+			int i;
+
+			/* composing chars */
+			for (i = 0; i < Screen_mco
+				      && i + 1 < VTERM_MAX_CHARS_PER_CELL; ++i)
+			{
+			    ScreenLinesC[i][off] = cell.chars[i + 1];
+			    if (cell.chars[i + 1] == 0)
+				break;
+			}
+			if (c >= 0x80 || (Screen_mco > 0
+						 && ScreenLinesC[0][off] != 0))
 			{
 			    ScreenLines[off] = ' ';
 			    ScreenLinesUC[off] = c;
@@ -3157,7 +3164,7 @@ f_term_sendkeys(typval_T *argvars, typval_T *rettv)
     while (*msg != NUL)
     {
 	send_keys_to_term(term, PTR2CHAR(msg), FALSE);
-	msg += MB_PTR2LEN(msg);
+	msg += MB_CPTR2LEN(msg);
     }
 }
 

src/testdir/test_terminal.vim

@@ -685,3 +685,60 @@ func Test_terminal_wall()
   exe buf . 'bwipe'
   unlet g:job
 endfunc
+
+func Test_terminal_composing_unicode()
+  let save_enc = &encoding
+  set encoding=utf-8
+
+  if has('win32')
+    let cmd = "cmd /K chcp 65001"
+    let lnum = [3, 6, 9]
+  else
+    let cmd = &shell
+    let lnum = [1, 3, 5]
+  endif
+
+  enew
+  let buf = term_start(cmd, {'curwin': bufnr('')})
+  let job = term_getjob(buf)
+  call term_wait(buf, 50)
+
+  " ascii + composing
+  let txt = "a\u0308bc"
+  call term_sendkeys(buf, "echo " . txt . "\r")
+  call term_wait(buf, 50)
+  call assert_match("echo " . txt, term_getline(buf, lnum[0]))
+  call assert_equal(txt, term_getline(buf, lnum[0] + 1))
+  let l = term_scrape(buf, lnum[0] + 1)
+  call assert_equal("a\u0308", l[0].chars)
+  call assert_equal("b", l[1].chars)
+  call assert_equal("c", l[2].chars)
+
+  " multibyte + composing
+  let txt = "\u304b\u3099\u304e\u304f\u3099\u3052\u3053\u3099"
+  call term_sendkeys(buf, "echo " . txt . "\r")
+  call term_wait(buf, 50)
+  call assert_match("echo " . txt, term_getline(buf, lnum[1]))
+  call assert_equal(txt, term_getline(buf, lnum[1] + 1))
+  let l = term_scrape(buf, lnum[1] + 1)
+  call assert_equal("\u304b\u3099", l[0].chars)
+  call assert_equal("\u304e", l[1].chars)
+  call assert_equal("\u304f\u3099", l[2].chars)
+  call assert_equal("\u3052", l[3].chars)
+  call assert_equal("\u3053\u3099", l[4].chars)
+
+  " \u00a0 + composing
+  let txt = "abc\u00a0\u0308"
+  call term_sendkeys(buf, "echo " . txt . "\r")
+  call term_wait(buf, 50)
+  call assert_match("echo " . txt, term_getline(buf, lnum[2]))
+  call assert_equal(txt, term_getline(buf, lnum[2] + 1))
+  let l = term_scrape(buf, lnum[2] + 1)
+  call assert_equal("\u00a0\u0308", l[3].chars)
+
+  call term_sendkeys(buf, "exit\r")
+  call WaitFor('job_status(job) == "dead"')
+  call assert_equal('dead', job_status(job))
+  bwipe!
+  let &encoding = save_enc
+endfunc

src/version.c

@@ -761,6 +761,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    1203,
 /**/
     1202,
 /**/