patch 8.1.0870: Vim doesn't use the new ConPTY support in Windows 10

Problem:    Vim doesn't use the new ConPTY support in Windows 10.
Solution:   Use ConPTY support, if available. (Nobuhiro Takasaki, closes #3794)

Author: brammool

runtime/doc/eval.txt

@@ -1,4 +1,4 @@
-*eval.txt*	For Vim version 8.1.  Last change: 2019 Jan 29
+*eval.txt*	For Vim version 8.1.  Last change: 2019 Feb 03
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -696,7 +696,7 @@ similar to -1. >
 	:let otherblob = myblob[:]	" make a copy of the Blob
 
 If the first index is beyond the last byte of the Blob or the second index is
-before the first index, the result is an empty list.  There is no error
+before the first index, the result is an empty Blob.  There is no error
 message.
 
 If the second index is equal to or greater than the length of the list the
@@ -9469,6 +9469,10 @@ term_start({cmd}, {options})				*term_start()*
 		   "ansi_colors"     A list of 16 color names or hex codes
 				     defining the ANSI palette used in GUI
 				     color modes.  See |g:terminal_ansi_colors|.
+		   "term_mode"	     (MS-Windows only): Specify which pty to
+				     use:
+					"winpty": Use winpty
+					"conpty": Use ConPTY (if available)
 
 		{only available when compiled with the |+terminal| feature}
 
@@ -10186,6 +10190,7 @@ cmdline_hist		Compiled with |cmdline-history| support.
 cmdline_info		Compiled with 'showcmd' and 'ruler' support.
 comments		Compiled with |'comments'| support.
 compatible		Compiled to be very Vi compatible.
+conpty			Platform where |ConPTY| can be used.
 cryptv			Compiled with encryption support |encryption|.
 cscope			Compiled with |cscope| support.
 cursorbind		Compiled with |cursorbind| (always true)

runtime/doc/options.txt

@@ -8054,6 +8054,23 @@ A jump table for the options with a short description can be found at |Q_op|.
 	Note that the "cterm" attributes are still used, not the "gui" ones.
 	NOTE: This option is reset when 'compatible' is set.
 
+						*'termmode'* *'tmod'*
+'termmode' 'tmod'	string  (default "")
+			local to window
+			{not in Vi, MS-Windows only}
+	Whether the window uses winpty or |ConPTY| as the virtual console.
+	When set before opening the terminal, it influences what pty is used.
+	When opening the terminal it will be set to the actually used pty.
+
+	Possible values are:
+	    ""		use ConPTY if possible, winpty otherwise
+	    "winpty"	use winpty, fail if not supported
+	    "conpty"	use |ConPTY|, fail if not supported
+
+	|ConPTY| support depends on the platform (Windows 10 October 2018
+	edition).  winpty support needs to be installed.  If neither is
+	supported then you cannot open a terminal window.
+
 						*'termwinscroll'* *'twsl'*
 'termwinscroll' 'twsl'	number	(default 10000)
 			local to buffer

runtime/doc/terminal.txt

@@ -228,7 +228,8 @@ Syntax ~
 					for Python "++eof=exit()".  Special
 					codes can be used like with `:map`,
 					e.g. "<C-Z>" for CTRL-Z.
-
+			++winpty	Use winpty as the virtual console.
+			++conpty	Use |ConPTY| as the virtual console.
 			If you want to use more options use the |term_start()|
 			function.
 			If you want to split the window vertically, use: >
@@ -410,6 +411,13 @@ Just put the files somewhere in your PATH.  You can set the 'winptydll' option
 to point to the right file, if needed.  If you have both the 32-bit and 64-bit
 version, rename to winpty32.dll and winpty64.dll to match the way Vim was
 build.
+							*ConPTY*
+On more recent versions of MS-Windows 10 (beginning with the "October 2018
+Update"), winpty is no longer required. On those versions, |:terminal| will use
+Windows' built-in support for hosting terminal applications, "ConPTY".  When
+ConPTY is in use, there may be rendering artifacts regarding ambiguous-width
+characters. If you encounter any such issues, set 'termmode' to winpty (which
+you then must have instlled).
 
 Environment variables are used to pass information to the running job:
     VIM_SERVERNAME	v:servername

src/channel.c

@@ -1720,11 +1720,7 @@ channel_get_all(channel_T *channel, ch_part_T part, int *outlen)
     char_u  *res;
     char_u  *p;
 
-    /* If there is only one buffer just get that one. */
-    if (head->rq_next == NULL || head->rq_next->rq_next == NULL)
-	return channel_get(channel, part, outlen);
-
-    /* Concatenate everything into one buffer. */
+    // Concatenate everything into one buffer.
     for (node = head->rq_next; node != NULL; node = node->rq_next)
 	len += node->rq_buflen;
     res = lalloc(len + 1, TRUE);
@@ -1738,7 +1734,7 @@ channel_get_all(channel_T *channel, ch_part_T part, int *outlen)
     }
     *p = NUL;
 
-    /* Free all buffers */
+    // Free all buffers
     do
     {
 	p = channel_get(channel, part, NULL);
@@ -1747,16 +1743,37 @@ channel_get_all(channel_T *channel, ch_part_T part, int *outlen)
 
     if (outlen != NULL)
     {
+	// Returning the length, keep NUL characters.
 	*outlen += len;
 	return res;
     }
 
-    /* turn all NUL into NL */
-    while (len > 0)
+    // Turn all NUL into NL, so that the result can be used as a string.
+    p = res;
+    while (p < res + len)
     {
-	--len;
-	if (res[len] == NUL)
-	    res[len] = NL;
+	if (*p == NUL)
+	    *p = NL;
+#ifdef WIN32
+	else if (*p == 0x1b)
+	{
+	    // crush the escape sequence OSC 0/1/2: ESC ]0;
+	    if (p + 3 < res + len
+		    && p[1] == ']'
+		    && (p[2] == '0' || p[2] == '1' || p[2] == '2')
+		    && p[3] == ';')
+	    {
+		// '\a' becomes a NL
+	        while (p < res + (len - 1) && *p != '\a')
+		    ++p;
+		// BEL is zero width characters, suppress display mistake
+		// ConPTY (after 10.0.18317) requires advance checking
+		if (p[-1] == NUL)
+		    p[-1] = 0x07;
+	    }
+	}
+#endif
+	++p;
     }
 
     return res;
@@ -4330,7 +4347,7 @@ channel_parse_messages(void)
 	    channel = first_channel;
 	    continue;
 	}
-	if (channel->ch_to_be_freed)
+	if (channel->ch_to_be_freed || channel->ch_killing)
 	{
 	    channel_free(channel);
 	    /* channel has been freed, start over */
@@ -4930,6 +4947,28 @@ get_job_options(typval_T *tv, jobopt_T *opt, int supported, int supported2)
 		opt->jo_set2 |= JO2_TERM_KILL;
 		opt->jo_term_kill = tv_get_string_chk(item);
 	    }
+	    else if (STRCMP(hi->hi_key, "term_mode") == 0)
+	    {
+		char_u *p;
+
+		if (!(supported2 & JO2_TERM_MODE))
+		    break;
+		opt->jo_set2 |= JO2_TERM_MODE;
+		p = tv_get_string_chk(item);
+		if (p == NULL)
+		{
+		    semsg(_(e_invargval), "term_mode");
+		    return FAIL;
+		}
+		// Allow empty string, "winpty", "conpty".
+		if (!(*p == NUL || STRCMP(p, "winpty") == 0
+					          || STRCMP(p, "conpty") == 0))
+		{
+		    semsg(_(e_invargval), "term_mode");
+		    return FAIL;
+		}
+		opt->jo_term_mode = p[0];
+	    }
 # if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS)
 	    else if (STRCMP(hi->hi_key, "ansi_colors") == 0)
 	    {
@@ -5440,6 +5479,16 @@ job_cleanup(job_T *job)
 	channel_need_redraw = TRUE;
     }
 
+    if (job->jv_channel != NULL
+	 && job->jv_channel->ch_anonymous_pipe && !job->jv_channel->ch_killing)
+    {
+	++safe_to_invoke_callback;
+	channel_free_contents(job->jv_channel);
+	job->jv_channel->ch_job = NULL;
+	job->jv_channel = NULL;
+	--safe_to_invoke_callback;
+    }
+
     // Do not free the job in case the close callback of the associated channel
     // isn't invoked yet and may get information by job_info().
     if (job->jv_refcount == 0 && !job_channel_still_useful(job))

src/evalfunc.c

@@ -6737,6 +6737,10 @@ f_has(typval_T *argvars, typval_T *rettv)
 #if defined(FEAT_TERMINAL) && defined(WIN3264)
 	else if (STRICMP(name, "terminal") == 0)
 	    n = terminal_enabled();
+#endif
+#if defined(FEAT_TERMINAL) && defined(WIN3264)
+	else if (STRICMP(name, "conpty") == 0)
+	    n = use_conpty();
 #endif
     }
 

src/globals.h

@@ -1432,7 +1432,8 @@ EXTERN char e_fsync[]		INIT(= N_("E667: Fsync failed"));
 	|| defined(DYNAMIC_ICONV) \
 	|| defined(DYNAMIC_GETTEXT) \
 	|| defined(DYNAMIC_MZSCHEME) \
-	|| defined(DYNAMIC_LUA)
+	|| defined(DYNAMIC_LUA) \
+	|| defined(FEAT_TERMINAL)
 EXTERN char e_loadlib[]	INIT(= N_("E370: Could not load library %s"));
 EXTERN char e_loadfunc[]	INIT(= N_("E448: Could not load library function %s"));
 #endif

src/option.c

@@ -253,6 +253,7 @@
 # define PV_TWK		OPT_WIN(WV_TWK)
 # define PV_TWS		OPT_WIN(WV_TWS)
 # define PV_TWSL	OPT_BUF(BV_TWSL)
+# define PV_TMOD	OPT_WIN(WV_TMOD)
 #endif
 #ifdef FEAT_SIGNS
 # define PV_SCL		OPT_WIN(WV_SCL)
@@ -2698,6 +2699,15 @@ static struct vimoption options[] =
 #else
 			    (char_u*)NULL, PV_NONE,
 			    {(char_u *)FALSE, (char_u *)FALSE}
+#endif
+			    SCTX_INIT},
+    {"termmode", "tmod",    P_STRING|P_ALLOCED|P_VI_DEF,
+#ifdef FEAT_TERMINAL
+			    (char_u *)VAR_WIN, PV_TMOD,
+			    {(char_u *)"", (char_u *)NULL}
+#else
+			    (char_u *)NULL, PV_NONE,
+			    {(char_u *)NULL, (char_u *)0L}
 #endif
 			    SCTX_INIT},
     {"termwinkey", "twk",   P_STRING|P_ALLOCED|P_RWIN|P_VI_DEF,
@@ -3208,6 +3218,9 @@ static char *(p_cot_values[]) = {"menu", "menuone", "longest", "preview", "noins
 #ifdef FEAT_SIGNS
 static char *(p_scl_values[]) = {"yes", "no", "auto", NULL};
 #endif
+#ifdef FEAT_TERMINAL
+static char *(p_tmod_values[]) = {"winpty", "conpty", "", NULL};
+#endif
 
 static void set_options_default(int opt_flags);
 static void set_string_default_esc(char *name, char_u *val, int escape);
@@ -3661,7 +3674,12 @@ set_init_1(int clean_arg)
 	    {
 		char	buf[50];
 
-		sprintf(buf, "cp%ld", (long)GetConsoleCP());
+		/* Win32 console: In ConPTY, GetConsoleCP() returns zero.
+		 * Use an alternative value. */
+		if (GetConsoleCP() == 0)
+		    sprintf(buf, "cp%ld", (long)GetACP());
+		else
+		    sprintf(buf, "cp%ld", (long)GetConsoleCP());
 		p_tenc = vim_strsave((char_u *)buf);
 		if (p_tenc != NULL)
 		{
@@ -7468,14 +7486,14 @@ did_set_string_option(
 #endif
 
 #ifdef FEAT_TERMINAL
-    /* 'termwinkey' */
+    // 'termwinkey'
     else if (varp == &curwin->w_p_twk)
     {
 	if (*curwin->w_p_twk != NUL
 				  && string_to_key(curwin->w_p_twk, TRUE) == 0)
 	    errmsg = e_invarg;
     }
-    /* 'termwinsize' */
+    // 'termwinsize'
     else if (varp == &curwin->w_p_tws)
     {
 	if (*curwin->w_p_tws != NUL)
@@ -7487,6 +7505,12 @@ did_set_string_option(
 		errmsg = e_invarg;
 	}
     }
+    // 'termmode'
+    else if (varp == &curwin->w_p_tmod)
+    {
+	if (check_opt_strings(*varp, p_tmod_values, FALSE) != OK)
+	    errmsg = e_invarg;
+    }
 #endif
 
 #ifdef FEAT_VARTABS
@@ -8838,7 +8862,7 @@ set_bool_option(
 	if (!has_vtp_working())
 	{
 	    p_tgc = 0;
-	    return (char_u*)N_("E954: 24-bit colors are not supported on this environment");
+	    return N_("E954: 24-bit colors are not supported on this environment");
 	}
 	if (is_term_win32())
 	    swap_tcap();
@@ -10928,6 +10952,7 @@ get_varp(struct vimoption *p)
 	case PV_TWK:    return (char_u *)&(curwin->w_p_twk);
 	case PV_TWS:    return (char_u *)&(curwin->w_p_tws);
 	case PV_TWSL:	return (char_u *)&(curbuf->b_p_twsl);
+	case PV_TMOD:	return (char_u *)&(curwin->w_p_tmod);
 #endif
 
 	case PV_AI:	return (char_u *)&(curbuf->b_p_ai);
@@ -11128,6 +11153,7 @@ copy_winopt(winopt_T *from, winopt_T *to)
 #ifdef FEAT_TERMINAL
     to->wo_twk = vim_strsave(from->wo_twk);
     to->wo_tws = vim_strsave(from->wo_tws);
+    to->wo_tmod = vim_strsave(from->wo_tmod);
 #endif
 #ifdef FEAT_FOLDING
     to->wo_fdc = from->wo_fdc;
@@ -11198,6 +11224,7 @@ check_winopt(winopt_T *wop UNUSED)
 #ifdef FEAT_TERMINAL
     check_string_option(&wop->wo_twk);
     check_string_option(&wop->wo_tws);
+    check_string_option(&wop->wo_tmod);
 #endif
 #ifdef FEAT_LINEBREAK
     check_string_option(&wop->wo_briopt);
@@ -11241,6 +11268,7 @@ clear_winopt(winopt_T *wop UNUSED)
 #ifdef FEAT_TERMINAL
     clear_string_option(&wop->wo_twk);
     clear_string_option(&wop->wo_tws);
+    clear_string_option(&wop->wo_tmod);
 #endif
 }
 

src/option.h

@@ -1112,6 +1112,7 @@ enum
 #ifdef FEAT_TERMINAL
     , WV_TWK
     , WV_TWS
+    , WV_TMOD
 #endif
     , WV_CRBIND
 #ifdef FEAT_LINEBREAK