patch 9.0.0912: libvterm with modifyOtherKeys level 2 does not match xterm

Problem:    libvterm with modifyOtherKeys level 2 does not match xterm.
Solution:   Adjust key code escape sequences to be the same as what xterm
            sends in modifyOtherKeys level 2 mode.  Check the value of
            no_reduce_keys before using it.

Author: brammool

src/getchar.c

@@ -2778,6 +2778,9 @@ handle_mapping(
 
 	    // If no termcode matched, try to include the modifier into the
 	    // key.  This is for when modifyOtherKeys is working.
+#ifdef FEAT_TERMINAL
+	    check_no_reduce_keys();  // may update the no_reduce_keys flag
+#endif
 	    if (keylen == 0 && !no_reduce_keys)
 	    {
 		keylen = check_simplify_modifier(max_mlen + 1);
@@ -3919,9 +3922,9 @@ getcmdkeycmd(
 	{
 	    // CTRL-V is followed by octal, hex or other characters, reverses
 	    // what AppendToRedobuffLit() does.
-	    no_reduce_keys = TRUE;  //  don't merge modifyOtherKeys
+	    ++no_reduce_keys;  //  don't merge modifyOtherKeys
 	    c1 = get_literal(TRUE);
-	    no_reduce_keys = FALSE;
+	    --no_reduce_keys;
 	}
 
 	if (got_int)

src/libvterm/src/keyboard.c

@@ -14,7 +14,7 @@ int vterm_is_modify_other_keys(VTerm *vt)
 void vterm_keyboard_unichar(VTerm *vt, uint32_t c, VTermModifier mod)
 {
   // VIM: added modifyOtherKeys support
-  if (vt->state->mode.modify_other_keys && mod != 0) {
+  if (vterm_is_modify_other_keys(vt) && mod != 0) {
     vterm_push_output_sprintf_ctrl(vt, C1_CSI, "27;%d;%d~", mod+1, c);
     return;
   }
@@ -184,7 +184,9 @@ void vterm_keyboard_key(VTerm *vt, VTermKey key, VTermModifier mod)
     break;
 
   case KEYCODE_LITERAL: case_LITERAL:
-    if(mod & (VTERM_MOD_SHIFT|VTERM_MOD_CTRL))
+    if (vterm_is_modify_other_keys(vt) && mod != 0)
+      vterm_push_output_sprintf_ctrl(vt, C1_CSI, "27;%d;%d~", mod+1, k.literal);
+    else if(mod & (VTERM_MOD_SHIFT|VTERM_MOD_CTRL))
       vterm_push_output_sprintf_ctrl(vt, C1_CSI, "%d;%du", k.literal, mod+1);
     else
       vterm_push_output_sprintf(vt, mod & VTERM_MOD_ALT ? ESC_S "%c" : "%c", k.literal);

src/misc2.c

@@ -1516,7 +1516,8 @@ find_special_key(
  * CTRL-2 is CTRL-@
  * CTRL-6 is CTRL-^
  * CTRL-- is CTRL-_
- * Also, <C-H> and <C-h> mean the same thing, always use "H".
+ * Also, unless no_reduce_keys is set then <C-H> and <C-h> mean the same thing,
+ * use "H".
  * Returns the possibly adjusted key.
  */
     int
@@ -1525,7 +1526,12 @@ may_adjust_key_for_ctrl(int modifiers, int key)
     if (modifiers & MOD_MASK_CTRL)
     {
 	if (ASCII_ISALPHA(key))
-	    return TOUPPER_ASC(key);
+	{
+#ifdef FEAT_TERMINAL
+	    check_no_reduce_keys();  // may update the no_reduce_keys flag
+#endif
+	    return no_reduce_keys == 0 ? TOUPPER_ASC(key) : key;
+	}
 	if (key == '2')
 	    return '@';
 	if (key == '6')

src/proto/terminal.pro

@@ -15,6 +15,7 @@ int term_try_stop_job(buf_T *buf);
 int term_check_timers(int next_due_arg, proftime_T *now);
 int term_in_normal_mode(void);
 void term_enter_job_mode(void);
+void check_no_reduce_keys(void);
 int send_keys_to_term(term_T *term, int c, int modmask, int typed);
 int terminal_is_active(void);
 cursorentry_T *term_get_cursor_shape(guicolor_T *fg, guicolor_T *bg);

src/terminal.c

@@ -2167,6 +2167,38 @@ term_enter_job_mode()
 #endif
 }
 
+/*
+ * When "modify_other_keys" is set then vgetc() should not reduce a key with
+ * modifiers into a basic key.  However, we may only find out after calling
+ * vgetc().  Therefore vgetorpeek() will call check_no_reduce_keys() to update
+ * "no_reduce_keys" before using it.
+ */
+typedef enum {
+    NRKS_NONE,	    // initial value
+    NRKS_CHECK,	    // modify_other_keys was off before calling vgetc()
+    NRKS_SET,	    // no_reduce_keys was incremented in term_vgetc() or
+		    // check_no_reduce_keys(), must be decremented.
+} reduce_key_state_T;
+
+static reduce_key_state_T  no_reduce_key_state = NRKS_NONE;
+
+    void
+check_no_reduce_keys(void)
+{
+    if (no_reduce_key_state != NRKS_CHECK
+	    || no_reduce_keys >= 1
+	    || curbuf->b_term == NULL
+	    || curbuf->b_term->tl_vterm == NULL)
+	return;
+
+    if (vterm_is_modify_other_keys(curbuf->b_term->tl_vterm))
+    {
+	// "modify_other_keys" was enabled while waiting.
+	no_reduce_key_state = NRKS_SET;
+	++no_reduce_keys;
+    }
+}
+
 /*
  * Get a key from the user with terminal mode mappings.
  * Note: while waiting a terminal may be closed and freed if the channel is
@@ -2177,21 +2209,32 @@ term_vgetc()
 {
     int c;
     int save_State = State;
-    int modify_other_keys = curbuf->b_term->tl_vterm == NULL ? FALSE
-			: vterm_is_modify_other_keys(curbuf->b_term->tl_vterm);
 
     State = MODE_TERMINAL;
     got_int = FALSE;
 #ifdef MSWIN
     ctrl_break_was_pressed = FALSE;
 #endif
-    if (modify_other_keys)
+
+    if (curbuf->b_term->tl_vterm != NULL
+		       && vterm_is_modify_other_keys(curbuf->b_term->tl_vterm))
+    {
 	++no_reduce_keys;
+	no_reduce_key_state = NRKS_SET;
+    }
+    else
+    {
+	no_reduce_key_state = NRKS_CHECK;
+    }
+
     c = vgetc();
     got_int = FALSE;
     State = save_State;
-    if (modify_other_keys)
+
+    if (no_reduce_key_state == NRKS_SET)
 	--no_reduce_keys;
+    no_reduce_key_state = NRKS_NONE;
+
     return c;
 }
 

src/testdir/keycode_check.json

@@ -1 +1 @@
-{"12xterm":{"Space":"20","version":"1b5b3e34313b3335363b3063","C-Tab":"1b5b32373b353b397e","A-Esc":"1b5b32373b333b32377e","C-Space":"1b5b32373b353b33327e","status":"","S-C-I":"1b5b32373b363b37337e","C-I":"1b5b32373b353b3130357e","S-Tab":"1b5b5a","Tab":"09","S-Space":"1b5b32373b323b33327e","A-Tab":"1b5b32373b333b397e","C-Esc":"1b5b32373b353b32377e","protocol":"mok2","A-Space":"1b5b32373b333b33327e","S-Esc":"1b","Esc":"1b"},"2kitty":{"Space":"20","version":"1b5b3e313b343030303b323163","C-Tab":"","A-Esc":"1b5b32373b313175","C-Space":"1b5b33323b3575","status":"1b5b3f3175","S-C-I":"1b5b3130353b3675","C-I":"1b5b3130353b3575","S-Tab":"1b5b393b3275","Tab":"09","S-Space":"20","A-Tab":"1b5b393b313175","C-Esc":"1b5b32373b3575","protocol":"kitty","A-Space":"1b5b33323b313175","S-Esc":"1b5b32373b3275","Esc":"1b5b323775"},"11xterm":{"Space":"20","version":"1b5b3e34313b3335363b3063","C-Tab":"09","A-Esc":"9b00","status":"","S-C-I":"09","C-I":"09","S-Tab":"1b5b5a","Tab":"09","S-Space":"20","A-Tab":"8900","C-Esc":"1b","protocol":"none","A-Space":"a000","S-Esc":"1b","Esc":"1b"}}
+{"12xterm":{"Space":"20","version":"1b5b3e34313b3335363b3063","C-Tab":"1b5b32373b353b397e","A-Esc":"1b5b32373b333b32377e","C-Space":"1b5b32373b353b33327e","status":"","S-C-I":"1b5b32373b363b37337e","C-I":"1b5b32373b353b3130357e","S-Tab":"1b5b5a","Tab":"09","S-Space":"1b5b32373b323b33327e","A-Tab":"1b5b32373b333b397e","C-Esc":"1b5b32373b353b32377e","protocol":"mok2","A-Space":"1b5b32373b333b33327e","S-Esc":"1b","Esc":"1b"},"libvterm":{"Space":"20","version":"1b5b3e303b3130303b3063","C-Tab":"1b5b32373b353b397e","A-Esc":"1b5b32373b333b32377e","C-Space":"1b5b32373b353b33327e","status":"","S-C-I":"1b5b32373b363b37337e","C-I":"1b5b32373b353b3130357e","S-Tab":"1b5b5a","Tab":"09","resource":"","A-Tab":"1b5b32373b333b397e","S-Space":"1b5b32373b323b33327e","C-Esc":"1b5b32373b353b32377e","protocol":"mok2","A-Space":"1b5b32373b333b33327e","S-Esc":"1b","Esc":"1b"},"2kitty":{"Space":"20","version":"1b5b3e313b343030303b323163","C-Tab":"","A-Esc":"1b5b32373b313175","C-Space":"1b5b33323b3575","status":"1b5b3f3175","S-C-I":"1b5b3130353b3675","C-I":"1b5b3130353b3575","S-Tab":"1b5b393b3275","Tab":"09","S-Space":"20","A-Tab":"1b5b393b313175","C-Esc":"1b5b32373b3575","protocol":"kitty","A-Space":"1b5b33323b313175","S-Esc":"1b5b32373b3275","Esc":"1b5b323775"},"11xterm":{"Space":"20","version":"1b5b3e34313b3335363b3063","C-Tab":"09","A-Esc":"9b00","status":"","S-C-I":"09","C-I":"09","S-Tab":"1b5b5a","Tab":"09","S-Space":"20","A-Tab":"8900","C-Esc":"1b","protocol":"none","A-Space":"a000","S-Esc":"1b","Esc":"1b"}}

src/testdir/keycode_check.vim

@@ -319,10 +319,10 @@ def DoTerm(name: string)
     ch_logfile('keylog-ignore', 'a')
     while 1
       sleep 100m
-      if !getchar(1)
+      if getchar(1) == 0
 	break
       endif
-      while getchar(1)
+      while getchar(1) != 0
 	getchar()
       endwhile
     endwhile

src/version.c

@@ -695,6 +695,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    912,
 /**/
     911,
 /**/