patch 8.2.2951: Vim9: cannot use heredoc for :python, :lua, etc.

Problem:    Vim9: cannot use heredoc in :def function for :python, :lua, etc.
Solution:   Concatenate the heredoc lines and pass them in the ISN_EXEC_SPLIT
            instruction.

Author: brammool

src/testdir/test_vim9_disassemble.vim

@@ -121,6 +121,23 @@ def Test_disassemble_exec_expr()
         res)
 enddef
 
+if has('python3')
+  def s:PyHeredoc()
+    python3 << EOF
+      print('hello')
+EOF
+  enddef
+
+  def Test_disassemble_python_heredoc()
+    var res = execute('disass s:PyHeredoc')
+    assert_match('<SNR>\d*_PyHeredoc.*' ..
+          "    python3 << EOF^@      print('hello')^@EOF\\_s*" ..
+          '\d EXEC_SPLIT     python3 << EOF^@      print(''hello'')^@EOF\_s*' ..
+          '\d RETURN 0',
+          res)
+  enddef
+endif
+
 def s:Substitute()
   var expr = "abc"
   :%s/a/\=expr/&g#c

src/testdir/test_vim9_func.vim

@@ -2758,5 +2758,33 @@ def Test_closing_brace_at_start_of_line()
   call CheckDefAndScriptSuccess(lines)
 enddef
 
+if has('python3')
+  def Test_python3_heredoc()
+    py3 << trim EOF
+      import vim
+      vim.vars['didit'] = 'yes'
+    EOF
+    assert_equal('yes', g:didit)
+
+    python3 << trim EOF
+      import vim
+      vim.vars['didit'] = 'again'
+    EOF
+    assert_equal('again', g:didit)
+  enddef
+endif
+
+" This messes up syntax highlight, keep near the end.
+if has('lua')
+  def Test_lua_heredoc()
+    g:d = {}
+    lua << trim EOF
+        x = vim.eval('g:d')
+        x['key'] = 'val'
+    EOF
+    assert_equal('val', g:d.key)
+  enddef
+endif
+
 
 " vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker

src/userfunc.c

@@ -631,8 +631,12 @@ get_function_body(
     char_u	*skip_until = NULL;
     int		ret = FAIL;
     int		is_heredoc = FALSE;
+    int		heredoc_concat_len = 0;
+    garray_T	heredoc_ga;
     char_u	*heredoc_trimmed = NULL;
 
+    ga_init2(&heredoc_ga, 1, 500);
+
     // Detect having skipped over comment lines to find the return
     // type.  Add NULL lines to keep the line count correct.
     sourcing_lnum_off = get_sourced_lnum(eap->getline, eap->cookie);
@@ -733,6 +737,20 @@ get_function_body(
 		    getline_options = vim9_function
 				? GETLINE_CONCAT_CONTBAR : GETLINE_CONCAT_CONT;
 		    is_heredoc = FALSE;
+
+		    if (heredoc_concat_len > 0)
+		    {
+			// Replace the starting line with all the concatenated
+			// lines.
+			ga_concat(&heredoc_ga, theline);
+			vim_free(((char_u **)(newlines->ga_data))[
+						      heredoc_concat_len - 1]);
+			((char_u **)(newlines->ga_data))[
+				  heredoc_concat_len - 1] = heredoc_ga.ga_data;
+			ga_init(&heredoc_ga);
+			heredoc_concat_len = 0;
+			theline += STRLEN(theline);  // skip the "EOF"
+		    }
 		}
 	    }
 	}
@@ -886,6 +904,8 @@ get_function_body(
 		    skip_until = vim_strnsave(p, skiptowhite(p) - p);
 		getline_options = GETLINE_NONE;
 		is_heredoc = TRUE;
+		if (eap->cmdidx == CMD_def)
+		    heredoc_concat_len = newlines->ga_len + 1;
 	    }
 
 	    // Check for ":cmd v =<< [trim] EOF"
@@ -928,10 +948,21 @@ get_function_body(
 	if (ga_grow(newlines, 1 + sourcing_lnum_off) == FAIL)
 	    goto theend;
 
-	// Copy the line to newly allocated memory.  get_one_sourceline()
-	// allocates 250 bytes per line, this saves 80% on average.  The cost
-	// is an extra alloc/free.
-	p = vim_strsave(theline);
+	if (heredoc_concat_len > 0)
+	{
+	    // For a :def function "python << EOF" concatenats all the lines,
+	    // to be used for the instruction later.
+	    ga_concat(&heredoc_ga, theline);
+	    ga_concat(&heredoc_ga, (char_u *)"\n");
+	    p = vim_strsave((char_u *)"");
+	}
+	else
+	{
+	    // Copy the line to newly allocated memory.  get_one_sourceline()
+	    // allocates 250 bytes per line, this saves 80% on average.  The
+	    // cost is an extra alloc/free.
+	    p = vim_strsave(theline);
+	}
 	if (p == NULL)
 	    goto theend;
 	((char_u **)(newlines->ga_data))[newlines->ga_len++] = p;
@@ -953,6 +984,7 @@ get_function_body(
 theend:
     vim_free(skip_until);
     vim_free(heredoc_trimmed);
+    vim_free(heredoc_ga.ga_data);
     need_wait_return |= saved_wait_return;
     return ret;
 }
@@ -1436,6 +1468,7 @@ deref_func_name(
 
     cc = name[*lenp];
     name[*lenp] = NUL;
+
     v = find_var(name, &ht, no_autoload);
     name[*lenp] = cc;
     if (v != NULL)

src/version.c

@@ -750,6 +750,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    2951,
 /**/
     2950,
 /**/

src/vim9.h

@@ -14,6 +14,7 @@
 typedef enum {
     ISN_EXEC,	    // execute Ex command line isn_arg.string
     ISN_EXECCONCAT, // execute Ex command from isn_arg.number items on stack
+    ISN_EXEC_SPLIT, // execute Ex command from isn_arg.string split at NL
     ISN_LEGACY_EVAL, // evaluate expression isn_arg.string with legacy syntax.
     ISN_ECHO,	    // echo isn_arg.echo.echo_count items on top of stack
     ISN_EXECUTE,    // execute Ex commands isn_arg.number items on top of stack

src/vim9compile.c

@@ -8668,6 +8668,29 @@ compile_exec(char_u *line, exarg_T *eap, cctx_T *cctx)
     return nextcmd;
 }
 
+/*
+ * A script command with heredoc, e.g.
+ *	ruby << EOF
+ *	   command
+ *	EOF
+ * Has been turned into one long line with NL characters by
+ * get_function_body():
+ *	ruby << EOF<NL>   command<NL>EOF
+ */
+    static char_u *
+compile_script(char_u *line, cctx_T *cctx)
+{
+    if (cctx->ctx_skip != SKIP_YES)
+    {
+	isn_T	*isn;
+
+	if ((isn = generate_instr(cctx, ISN_EXEC_SPLIT)) == NULL)
+	    return NULL;
+	isn->isn_arg.string = vim_strsave(line);
+    }
+    return (char_u *)"";
+}
+
 
 /*
  * :s/pat/repl/
@@ -9480,18 +9503,28 @@ compile_def_function(
 		    line = (char_u *)"";
 		    break;
 
-	    default:
-		    if (cctx.ctx_skip == SKIP_YES)
-		    {
-			// We don't check for a next command here.
-			line = (char_u *)"";
-		    }
-		    else
-		    {
-			// Not recognized, execute with do_cmdline_cmd().
-			ea.arg = p;
+	    case CMD_lua:
+	    case CMD_mzscheme:
+	    case CMD_perl:
+	    case CMD_py3:
+	    case CMD_python3:
+	    case CMD_python:
+	    case CMD_pythonx:
+	    case CMD_ruby:
+	    case CMD_tcl:
+		    ea.arg = p;
+		    if (vim_strchr(line, '\n') == NULL)
 			line = compile_exec(line, &ea, &cctx);
-		    }
+		    else
+			// heredoc lines have been concatenated with NL
+			// characters in get_function_body()
+			line = compile_script(line, &cctx);
+		    break;
+
+	    default:
+		    // Not recognized, execute with do_cmdline_cmd().
+		    ea.arg = p;
+		    line = compile_exec(line, &ea, &cctx);
 		    break;
 	}
 nextline:
@@ -9674,6 +9707,7 @@ delete_instr(isn_T *isn)
     {
 	case ISN_DEF:
 	case ISN_EXEC:
+	case ISN_EXEC_SPLIT:
 	case ISN_LEGACY_EVAL:
 	case ISN_LOADAUTO:
 	case ISN_LOADB:

src/vim9execute.c

@@ -1213,6 +1213,37 @@ get_script_svar(scriptref_T *sref, ectx_T *ectx)
     return sv;
 }
 
+/*
+ * Function passed to do_cmdline() for splitting a script joined by NL
+ * characters.
+ */
+    static char_u *
+get_split_sourceline(
+	int c UNUSED,
+	void *cookie,
+	int indent UNUSED,
+	getline_opt_T options UNUSED)
+{
+    source_cookie_T	*sp = (source_cookie_T *)cookie;
+    char_u		*p;
+    char_u		*line;
+
+    if (*sp->nextline == NUL)
+	return NULL;
+    p = vim_strchr(sp->nextline, '\n');
+    if (p == NULL)
+    {
+	line = vim_strsave(sp->nextline);
+	sp->nextline += STRLEN(sp->nextline);
+    }
+    else
+    {
+	line = vim_strnsave(sp->nextline, p - sp->nextline);
+	sp->nextline = p + 1;
+    }
+    return line;
+}
+
 /*
  * Execute a function by "name".
  * This can be a builtin function, user function or a funcref.
@@ -1425,6 +1456,24 @@ exec_instructions(ectx_T *ectx)
 		}
 		break;
 
+	    // execute Ex command line split at NL characters.
+	    case ISN_EXEC_SPLIT:
+		{
+		    source_cookie_T cookie;
+
+		    SOURCING_LNUM = iptr->isn_lnum;
+		    CLEAR_FIELD(cookie);
+		    cookie.sourcing_lnum = iptr->isn_lnum - 1;
+		    cookie.nextline = iptr->isn_arg.string;
+		    if (do_cmdline(get_split_sourceline(0, &cookie, 0, 0),
+				get_split_sourceline, &cookie,
+				   DOCMD_VERBOSE|DOCMD_NOWAIT|DOCMD_KEYTYPED)
+									== FAIL
+				|| did_emsg)
+			goto on_error;
+		}
+		break;
+
 	    // Evaluate an expression with legacy syntax, push it onto the
 	    // stack.
 	    case ISN_LEGACY_EVAL:
@@ -4536,6 +4585,9 @@ list_instructions(char *pfx, isn_T *instr, int instr_count, ufunc_T *ufunc)
 	    case ISN_EXEC:
 		smsg("%s%4d EXEC %s", pfx, current, iptr->isn_arg.string);
 		break;
+	    case ISN_EXEC_SPLIT:
+		smsg("%s%4d EXEC_SPLIT %s", pfx, current, iptr->isn_arg.string);
+		break;
 	    case ISN_LEGACY_EVAL:
 		smsg("%s%4d EVAL legacy %s", pfx, current,
 							 iptr->isn_arg.string);