patch 8.2.1439: tiny and small builds have no test coverage

Problem:    Tiny and small builds have no test coverage.
Solution:   Restore tests that do not depend on the +eval feature.
            (Ken Takata, closes #6696)

Author: brammool

.travis.yml

@@ -7,11 +7,11 @@ env:
 _anchors:
   envs:
     - &tiny-nogui
-      FEATURES=tiny CONFOPT="--disable-gui"
+      FEATURES=tiny TEST=testtiny CONFOPT="--disable-gui"
     - &tiny
-      FEATURES=tiny
+      FEATURES=tiny TEST=testtiny
     - &small
-      FEATURES=small
+      FEATURES=small TEST=testtiny
     - &normal
       FEATURES=normal
     - &linux-huge

Filelist

@@ -158,9 +158,7 @@ SRC_ALL =	\
 		src/testdir/gen_opt_test.vim \
 		src/testdir/README.txt \
 		src/testdir/Make_all.mak \
-		src/testdir/dotest.in \
-		src/testdir/test1.in \
-		src/testdir/test77a.in \
+		src/testdir/*.in \
 		src/testdir/*.py \
 		src/testdir/lsan-suppress.txt \
 		src/testdir/sautest/autoload/*.vim \
@@ -179,7 +177,7 @@ SRC_ALL =	\
 		src/testdir/summarize.vim \
 		src/testdir/term_util.vim \
 		src/testdir/view_util.vim \
--		src/testdir/test1.ok \
+		src/testdir/test[0-9]*.ok \
 		src/testdir/test77a.ok \
 		src/testdir/test83-tags? \
 		src/testdir/test77a.com \

Makefile

@@ -32,7 +32,7 @@ first:
 
 # Some make programs use the last target for the $@ default; put the other
 # targets separately to always let $@ expand to "first" by default.
-all install uninstall tools config configure reconfig proto depend lint tags types test scripttests test_libvterm unittests testclean clean distclean:
+all install uninstall tools config configure reconfig proto depend lint tags types test scripttests testtiny test_libvterm unittests testclean clean distclean:
 	@if test ! -f src/auto/config.mk; then \
 		cp src/config.mk.dist src/auto/config.mk; \
 	fi

runtime/doc/testing.txt

@@ -20,11 +20,17 @@ and for testing plugins.
 Vim can be tested after building it, usually with "make test".
 The tests are located in the directory "src/testdir".
 
+There are two types of tests added over time:
+	test20.in		oldest, only for tiny and small builds
+	test_something.vim	new style tests
+
 						*new-style-testing*
 New tests should be added as new style tests.  The test scripts are named
 test_<feature>.vim (replace <feature> with the feature under test). These use
 functions such as |assert_equal()| to keep the test commands and the expected
 result in one place.
+						*old-style-testing*
+These tests are used only for testing Vim without the |+eval| feature.
 
 Find more information in the file src/testdir/README.txt.
 

src/Make_mvc.mak

@@ -1464,22 +1464,32 @@ cmdidxs: ex_cmds.h
 
 test:
 	cd testdir
-	$(MAKE) /NOLOGO -f Make_dos.mak win32
+	$(MAKE) /NOLOGO -f Make_dos.mak
 	cd ..
 
 testgvim:
 	cd testdir
-	$(MAKE) /NOLOGO -f Make_dos.mak VIMPROG=..\gvim win32
+	$(MAKE) /NOLOGO -f Make_dos.mak VIMPROG=..\gvim
+	cd ..
+
+testtiny:
+	cd testdir
+	$(MAKE) /NOLOGO -f Make_dos.mak tiny
+	cd ..
+
+testgvimtiny:
+	cd testdir
+	$(MAKE) /NOLOGO -f Make_dos.mak tiny VIMPROG=..\gvim
 	cd ..
 
 testclean:
 	cd testdir
 	$(MAKE) /NOLOGO -f Make_dos.mak clean
 	cd ..
 
-# Run test1 to bootstrap tests
+# Run individual OLD style test.
 # These do not depend on the executable, compile it when needed.
-$(SCRIPTS_FIRST:.out=):
+$(SCRIPTS_TINY):
 	cd testdir
 	- if exist $@.out del $@.out
 	$(MAKE) /NOLOGO -f Make_dos.mak VIMPROG=..\$(VIMTESTTARGET) nolog

src/Makefile

@@ -2258,6 +2258,9 @@ scripttests:
 testgui:
 	cd testdir; $(MAKE) -f Makefile $(GUI_TESTTARGET) VIMPROG=../$(VIMTARGET) GUI_FLAG=-g $(GUI_TESTARG) SCRIPTSOURCE=../$(SCRIPTSOURCE)
 
+testtiny:
+	cd testdir; $(MAKE) -f Makefile tiny VIMPROG=../$(VIMTARGET) SCRIPTSOURCE=../$(SCRIPTSOURCE)
+
 benchmark:
 	cd testdir; $(MAKE) -f Makefile benchmark VIMPROG=../$(VIMTARGET) SCRIPTSOURCE=../$(SCRIPTSOURCE)
 
@@ -2291,9 +2294,9 @@ test_libvterm:
 			CC="$(CC)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"; \
 	fi
 
-# Run test1, used to bootstrap tests.
-# This does not depend on the executable, compile first it when needed.
-test1:
+# Run individual OLD style test.
+# These do not depend on the executable, compile it when needed.
+$(SCRIPTS_TINY):
 	cd testdir; rm -f $@.out; $(MAKE) -f Makefile $@.out VIMPROG=../$(VIMTESTTARGET) $(GUI_TESTARG) SCRIPTSOURCE=../$(SCRIPTSOURCE)
 
 # Run individual NEW style test.

src/testdir/Make_all.mak

@@ -7,8 +7,26 @@
 NO_PLUGINS = --noplugin --not-a-term
 NO_INITS = -U NONE $(NO_PLUGINS)
 
-# The first script creates small.vim.
-SCRIPTS_FIRST = test1.out
+# Tests for tiny and small builds.
+SCRIPTS_TINY = \
+	test20 \
+	test21 \
+	test22 \
+	test23 \
+	test24 \
+	test25 \
+	test26 \
+	test27
+
+SCRIPTS_TINY_OUT = \
+	test20.out \
+	test21.out \
+	test22.out \
+	test23.out \
+	test24.out \
+	test25.out \
+	test26.out \
+	test27.out
 
 # Tests for Vim9 script.
 TEST_VIM9 = \
@@ -25,6 +43,9 @@ TEST_VIM9_RES = \
 	test_vim9_func.res \
 	test_vim9_script.res
 
+# Benchmark scripts.
+SCRIPTS_BENCH = test_bench_regexp.res
+
 # Individual tests, including the ones part of test_alot.
 # Please keep sorted up to test_alot.
 NEW_TESTS = \

src/testdir/Make_amiga.mak

@@ -9,9 +9,11 @@ default: nongui
 
 include Make_all.mak
 
+SCRIPTS = $(SCRIPTS_TINY_OUT)
+
 .SUFFIXES: .in .out .res .vim
 
-nongui:	/tmp $(SCRIPTS_FIRST)
+nongui:	/tmp $(SCRIPTS)
 	csh -c echo ALL DONE
 
 clean:

src/testdir/Make_dos.mak

@@ -9,56 +9,23 @@ default: nongui
 
 !include Make_all.mak
 
-TEST_OUTFILES = $(SCRIPTS_FIRST)
+# Explicit dependencies.
+test_options.res test_alot.res: opt_test.vim
+
+TEST_OUTFILES = $(SCRIPTS_TINY_OUT)
 DOSTMP = dostmp
 DOSTMP_OUTFILES = $(TEST_OUTFILES:test=dostmp\test)
 DOSTMP_INFILES = $(DOSTMP_OUTFILES:.out=.in)
 
 .SUFFIXES: .in .out .res .vim
 
-nongui:	nolog $(SCRIPTS_FIRST) newtests report
+nongui:	nolog tinytests newtests report
 
-small:	nolog report
+gui:	nolog tinytests newtests report
 
-gui:	nolog $(SCRIPTS_FIRST) newtests report
+tiny:	nolog tinytests report
 
-win32:	nolog $(SCRIPTS_FIRST) newtests report
-
-# Copy the input files to dostmp, changing the fileformat to dos.
-$(DOSTMP_INFILES): $(*B).in
-	if not exist $(DOSTMP)\NUL md $(DOSTMP)
-	if exist $@ del $@
-	$(VIMPROG) -u dos.vim $(NO_INITS) "+set ff=dos|f $@|wq" $(*B).in
-
-# For each input file dostmp/test99.in run the tests.
-# This moves test99.in to test99.in.bak temporarily.
-$(TEST_OUTFILES): $(DOSTMP)\$(*B).in
-	-@if exist test.out DEL test.out
-	-@if exist $(DOSTMP)\$(*B).out DEL $(DOSTMP)\$(*B).out
-	move $(*B).in $(*B).in.bak > nul
-	copy $(DOSTMP)\$(*B).in $(*B).in > nul
-	copy $(*B).ok test.ok > nul
-	$(VIMPROG) -u dos.vim $(NO_INITS) -s dotest.in $(*B).in
-	-@if exist test.out MOVE /y test.out $(DOSTMP)\$(*B).out > nul
-	-@if exist $(*B).in.bak move /y $(*B).in.bak $(*B).in > nul
-	-@if exist test.ok del test.ok
-	-@if exist Xdir1 rd /s /q Xdir1
-	-@if exist Xfind rd /s /q Xfind
-	-@if exist XfakeHOME rd /s /q XfakeHOME
-	-@del X*
-	-@if exist viminfo del viminfo
-	$(VIMPROG) -u dos.vim $(NO_INITS) "+set ff=unix|f test.out|wq" \
-		$(DOSTMP)\$(*B).out
-	@diff test.out $*.ok & if errorlevel 1 \
-		( move /y test.out $*.failed > nul \
-		 & del $(DOSTMP)\$(*B).out \
-		 & echo $* FAILED >> test.log ) \
-		else ( move /y test.out $*.out > nul )
-
-# Must run test1 first to create small.vim.
-# This rule must come after the one that copies the input files to dostmp to
-# allow for running an individual test.
-$(SCRIPTS) $(SCRIPTS_GUI) $(SCRIPTS_WIN32) $(NEW_TESTS_RES): $(SCRIPTS_FIRST)
+benchmark: $(SCRIPTS_BENCH)
 
 report:
 	@rem without the +eval feature test_result.log is a copy of test.log
@@ -71,10 +38,24 @@ report:
 	@if exist test.log ( echo TEST FAILURE & exit /b 1 ) \
 		else ( echo ALL DONE )
 
+
+# Execute an individual new style test, e.g.:
+# 	nmake -f Make_dos.mak test_largefile
+$(NEW_TESTS):
+	-if exist $@.res del $@.res
+	-if exist test.log del test.log
+	-if exist messages del messages
+	@$(MAKE) -nologo -f Make_dos.mak $@.res VIMPROG=$(VIMPROG)
+	@type messages
+	@if exist test.log exit 1
+
+
+# Delete files that may interfere with running tests.  This includes some files
+# that may result from working on the tests, not only from running them.
 clean:
-	-del *.out
-	-del *.failed
-	-del *.res
+	-if exist *.out $(DEL) *.out
+	-if exist *.failed $(DEL) *.failed
+	-if exist *.res $(DEL) *.res
 	-if exist $(DOSTMP) rd /s /q $(DOSTMP)
 	-if exist test.in del test.in
 	-if exist test.ok del test.ok
@@ -85,8 +66,8 @@ clean:
 	-if exist Xdir1 rd /s /q Xdir1
 	-if exist Xfind rd /s /q Xfind
 	-if exist XfakeHOME rd /s /q XfakeHOME
-	-del X*
-	-for /d %i in (X*) do @rmdir /s/q %i
+	-if exist X* $(DEL) X*
+	-for /d %i in (X*) do @rd /s/q %i
 	-if exist viminfo del viminfo
 	-if exist test.log del test.log
 	-if exist test_result.log del test_result.log
@@ -99,14 +80,41 @@ nolog:
 	-if exist test_result.log del test_result.log
 	-if exist messages del messages
 
-benchmark: test_bench_regexp.res
 
-test_bench_regexp.res: test_bench_regexp.vim
-	-if exist benchmark.out del benchmark.out
-	@echo $(VIMPROG) > vimcmd
-	$(VIMPROG) -u NONE $(NO_INITS) -S runtest.vim $*.vim
-	@del vimcmd
-	@IF EXIST benchmark.out ( type benchmark.out )
+# Tiny tests.  Works even without the +eval feature.
+tinytests: $(SCRIPTS_TINY_OUT)
+
+# Copy the input files to dostmp, changing the fileformat to dos.
+$(DOSTMP_INFILES): $(*B).in
+	if not exist $(DOSTMP)\NUL md $(DOSTMP)
+	if exist $@ del $@
+	$(VIMPROG) -u dos.vim $(NO_INITS) "+set ff=dos|f $@|wq" $(*B).in
+
+# For each input file dostmp/test99.in run the tests.
+# This moves test99.in to test99.in.bak temporarily.
+$(TEST_OUTFILES): $(DOSTMP)\$(*B).in
+	-@if exist test.out DEL test.out
+	-@if exist $(DOSTMP)\$(*B).out DEL $(DOSTMP)\$(*B).out
+	move $(*B).in $(*B).in.bak > nul
+	copy $(DOSTMP)\$(*B).in $(*B).in > nul
+	copy $(*B).ok test.ok > nul
+	$(VIMPROG) -u dos.vim $(NO_INITS) -s dotest.in $(*B).in
+	-@if exist test.out MOVE /y test.out $(DOSTMP)\$(*B).out > nul
+	-@if exist $(*B).in.bak move /y $(*B).in.bak $(*B).in > nul
+	-@if exist test.ok del test.ok
+	-@if exist Xdir1 rd /s /q Xdir1
+	-@if exist Xfind rd /s /q Xfind
+	-@if exist XfakeHOME rd /s /q XfakeHOME
+	-@del X*
+	-@if exist viminfo del viminfo
+	$(VIMPROG) -u dos.vim $(NO_INITS) "+set ff=unix|f test.out|wq" \
+		$(DOSTMP)\$(*B).out
+	@diff test.out $*.ok & if errorlevel 1 \
+		( move /y test.out $*.failed > nul \
+		 & del $(DOSTMP)\$(*B).out \
+		 & echo $* FAILED >> test.log ) \
+		else ( move /y test.out $*.out > nul )
+
 
 # New style of tests uses Vim script with assert calls.  These are easier
 # to write and a lot easier to read and debug.
@@ -132,7 +140,12 @@ test_gui_init.res: test_gui_init.vim
 	$(VIMPROG) -u gui_preinit.vim -U gui_init.vim $(NO_PLUGINS) -S runtest.vim $*.vim
 	@del vimcmd
 
-test_options.res test_alot.res: opt_test.vim
-
 opt_test.vim: ../optiondefs.h gen_opt_test.vim
 	$(VIMPROG) -u NONE -S gen_opt_test.vim --noplugin --not-a-term ../optiondefs.h
+
+test_bench_regexp.res: test_bench_regexp.vim
+	-if exist benchmark.out del benchmark.out
+	@echo $(VIMPROG) > vimcmd
+	$(VIMPROG) -u NONE $(NO_INITS) -S runtest.vim $*.vim
+	@del vimcmd
+	@IF EXIST benchmark.out ( type benchmark.out )