patch 9.1.1110: Vim tests are slow and flaky

Problem:  Vim tests are slow and flaky at the same time due to reliance
          on timeouts which are unreliable.
Solution: improve Vim test performance and reduce flakiness
          (Yee Cheng Chin)

A lot of Vim tests currently rely on waiting a specific amount of time
before asserting a condition. This is bad because 1) it is slow, as the
timeout is hardcoded, 2) it's unreliable as a resource-starved runner
may overshoot the timeout. Also, there are a lot of builtin sleep
commands in commonly used utilities like VerifyScreenDump and WaitFor()
which leads to a lot of unnecessary idle time.

Fix these issues by doing the following:
1. Make utilities like VerifyScreenDump and WaitFor use the lowest wait
   time possible (1 ms). This essentially turns it into a spin wait. On
   fast machines, these will finish very quickly. For existing tests
   that had an implicit reliance on the old timeouts (e.g.
   VerifyScreenDump had a 50ms wait before), fix the tests to wait that
   specific amount explicitly.
2. Fix tests that sleep or wait for long amounts of time to instead
   explicitly use a callback mechanism to be notified when a child
   terminal job has finished. This allows the test to only take as much
   time as possible instead of having to hard code an unreliable
   timeout.

With these fixes, tests should 1) completely quickly on fast machines,
and 2) on slow machines they will still run to completion albeit slowly.
Note that previoulsy both were not true. The hardcoded timeouts meant
that on fast machines the tests were mostly idling wasting time, whereas
on slow machines, the timeouts often were not generous enough to allow
them to run to completion.

closes: #16615

Signed-off-by: Yee Cheng Chin <[email protected]>
Signed-off-by: Christian Brabandt <[email protected]>

Author: ychin

runtime/doc/terminal.txt

@@ -1,4 +1,4 @@
-*terminal.txt*	For Vim version 9.1.  Last change: 2025 Jan 11
+*terminal.txt*	For Vim version 9.1.  Last change: 2025 Feb 13
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -999,7 +999,8 @@ term_wait({buf} [, {time}])					*term_wait()*
 		Wait for pending updates of {buf} to be handled.
 		{buf} is used as with |term_getsize()|.
 		{time} is how long to wait for updates to arrive in msec.  If
-		not set then 10 msec will be used.
+		not set then 10 msec will be used.  Queued messages will also
+		be processed similar to |:sleep|.
 
 		Can also be used as a |method|: >
 			GetBufnr()->term_wait()

runtime/doc/various.txt

@@ -1,4 +1,4 @@
-*various.txt*   For Vim version 9.1.  Last change: 2024 Dec 16
+*various.txt*   For Vim version 9.1.  Last change: 2025 Feb 13
 
 
 		  VIM REFERENCE MANUAL    by Bram Moolenaar
@@ -765,9 +765,8 @@ K			Run a program to lookup the keyword under the
 			MS-Windows).  "gs" stands for "goto sleep".
 			While sleeping the cursor is positioned in the text,
 			if at a visible position.
-			Also process the received netbeans messages. {only
-			available when compiled with the |+netbeans_intg|
-			feature}
+			Queued messages and timers (|+timers|) are processed
+			during the sleep as well.
 
 							*:sl!* *:sleep!*
 :[N]sl[eep]! [N][m]	Same as above, but hide the cursor.

src/testdir/crash/ex_redraw_crash

@@ -265,6 +265,9 @@ func RunTheTest(test)
   " buffers.
   %bwipe!
 
+  " Clear all children notifications in case there are stale ones left
+  let g:child_notification = 0
+
   " The test may change the current directory. Save and restore the
   " directory after executing the test.
   let save_cwd = getcwd()

src/testdir/runtest.vim

@@ -52,14 +52,14 @@ func VerifyScreenDump(buf, filename, options, ...)
   let filter = 'dumps/' . a:filename . '.vim'
   let testfile = 'failed/' . a:filename . '.dump'
 
-  let max_loops = get(a:options, 'wait', 1000) / 10
+  let max_loops = get(a:options, 'wait', 1000) / 1
 
   " Starting a terminal to make a screendump is always considered flaky.
   let g:test_is_flaky = 1
   let g:giveup_same_error = 0
 
   " wait for the pending updates to be handled.
-  call TermWait(a:buf)
+  call TermWait(a:buf, 0)
 
   " Redraw to execute the code that updates the screen.  Otherwise we get the
   " text and attributes only from the internal buffer.
@@ -80,8 +80,8 @@ func VerifyScreenDump(buf, filename, options, ...)
 
   let i = 0
   while 1
-    " leave some time for updating the original window
-    sleep 50m
+    " leave a bit of time for updating the original window while we spin wait.
+    sleep 1m
     call delete(testfile)
     call term_dumpwrite(a:buf, testfile, a:options)
 

src/testdir/screendump.vim

@@ -136,6 +136,34 @@ func s:kill_server(cmd)
   endif
 endfunc
 
+" Callback function to be invoked by a child terminal job. The parent could
+" then wait for the notification using WaitForChildNotification()
+let g:child_notification = 0
+func Tapi_notify_parent(bufnum, arglist)
+  let g:child_notification = 1
+endfunc
+
+" Generates a command that we can pass to a terminal job that it uses to
+" notify us. Argument 'escape' will specify whether to escape the double
+" quote.
+func TermNotifyParentCmd(escape)
+  call assert_false(has("win32"), 'Windows does not support terminal API right now. Use another method to synchronize timing.')
+  let cmd = '\033]51;["call", "Tapi_notify_parent", []]\007'
+  if a:escape
+    return escape(cmd, '"')
+  endif
+  return cmd
+endfunc
+
+" Wait for a child process to notify us. This allows us to sequence events in
+" conjunction with the child. Currently the only supported notification method
+" is for a terminal job to call Tapi_notify_parent() using terminal API.
+func WaitForChildNotification(...)
+  let timeout = get(a:000, 0, 5000)
+  call WaitFor({-> g:child_notification == 1}, timeout)
+  let g:child_notification = 0
+endfunc
+
 " Wait for up to five seconds for "expr" to become true.  "expr" can be a
 " stringified expression to evaluate, or a funcref without arguments.
 " Using a lambda works best.  Example:
@@ -162,7 +190,7 @@ endfunc
 " A second argument can be used to specify a different timeout in msec.
 "
 " Return zero for success, one for failure (like the assert function).
-func WaitForAssert(assert, ...)
+func g:WaitForAssert(assert, ...)
   let timeout = get(a:000, 0, 5000)
   if s:WaitForCommon(v:null, a:assert, timeout) < 0
     return 1
@@ -200,11 +228,11 @@ func s:WaitForCommon(expr, assert, timeout)
       call remove(v:errors, -1)
     endif
 
-    sleep 10m
+    sleep 1m
     if exists('*reltimefloat')
       let slept = float2nr(reltimefloat(reltime(start)) * 1000)
     else
-      let slept += 10
+      let slept += 1
     endif
   endwhile
 

src/testdir/shared.vim

@@ -157,14 +157,14 @@ func StopVimInTerminal(buf, kill = 1)
   call assert_equal("running", term_getstatus(a:buf))
 
   " Wait for all the pending updates to terminal to complete
-  call TermWait(a:buf)
+  call TermWait(a:buf, 1)
 
   " CTRL-O : works both in Normal mode and Insert mode to start a command line.
   " In Command-line it's inserted, the CTRL-U removes it again.
   call term_sendkeys(a:buf, "\<C-O>:\<C-U>qa!\<cr>")
 
   " Wait for all the pending updates to terminal to complete
-  call TermWait(a:buf)
+  call TermWait(a:buf, 1)
 
   " Wait for the terminal to end.
   call WaitForAssert({-> assert_equal("finished", term_getstatus(a:buf))})

src/testdir/term_util.vim

@@ -36,6 +36,7 @@ func Test_balloon_eval_term()
   call TermWait(buf, 50)
   call term_sendkeys(buf, 'll')
   call term_sendkeys(buf, ":call Trigger()\<CR>")
+  sleep 150m " Wait for balloon to show up (100ms balloondelay time)
   call VerifyScreenDump(buf, 'Test_balloon_eval_term_01', {})
 
   " Make sure the balloon still shows after 'updatetime' passed and CursorHold