-
Notifications
You must be signed in to change notification settings - Fork 174
/
run_tests
executable file
·368 lines (324 loc) · 9.1 KB
/
run_tests
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
#!/usr/bin/env bash
AbsDirOf() {
if [ -d "$1" ]; then
( cd "$1" && pwd -P )
else
( cd $(dirname "$1") && pwd -P )
fi
}
HOMEDIR=$(AbsDirOf "$0")
pushd ${HOMEDIR}
BASEDIR=${HOMEDIR}/${VIMSPECTOR_TEST_BASE:-test-base}
INSTALL=0
UPDATE=0
INSTALLER_ARGS=''
VIM_EXE='vim'
SCREENCOLS=160
SCREENROWS=48
ASCIINEMA="python3 -m asciinema"
TEST_UNTIL_FAILURE=0
NO_QUIT_ON_FAIL=0
# 1 is stdout
out_fd=1
while [ -n "$1" ]; do
case "$1" in
"--basedir"|"--base-dir"|"--test-base")
shift
BASEDIR=$(AbsDirOf "$1")
shift
;;
"--until-failure")
shift
TEST_UNTIL_FAILURE=1
;;
"--install")
INSTALL=1
shift
;;
"--install-method")
shift
INSTALL=$1
shift
;;
"--update"|"--upgrade")
UPDATE=1
shift
;;
"--no-quit-on-fail")
NO_QUIT_ON_FAIL=1
shift
;;
"--report")
shift
VIMSPECTOR_TEST_STDOUT=$1
shift
;;
"--exe")
shift
VIM_EXE=$1
shift
;;
"--failfast"|"--no-retry")
shift
export TEST_NO_RETRY=1
;;
"--quick")
shift
export TEST_NO_RETRY=1
SKIP_BUILD=1
;;
"--skip-build")
shift
SKIP_BUILD=1
;;
"--quiet")
shift
# send the output to /dev/null
# https://stackoverflow.com/a/47553900
# Note we can't use {out_fd} here because the bash version in CI is too
# old on macOS
out_fd=3
exec 3>/dev/null
INSTALLER_ARGS="${INSTALLER_ARGS} --quiet"
;;
"--gdb")
shift
export VIMSPECTOR_MIMODE=gdb
;;
"--lldb")
shift
export VIMSPECTOR_MIMODE=lldb
;;
"--")
shift
break
;;
"--help")
shift
echo "$(basename $0) [--basedir <basedir>] [--report output] [--exe <executable>] [--skip-build] [--quiet] [--install]|[--install-method script|vim] <optional list of tests in form file:func>"
echo ""
echo " --basedir <basedir> path to runtime directory like the option to install_gadget.py"
echo " --install run install_gadget.py, useful with --basedir"
echo " --report <messages|all> which logs to dump to stdout after a test"
echo " --exe <executable> choose vim executable, default 'vim'"
echo " --skip-build do not (re)build test programs"
echo " --quiet suppress vim's stdout"
echo "e.g.: "
echo " - run all tests: $0"
echo " - run specific tests script: $0 signature_help.test.vim"
echo " - run specific tests fun: $0 signature_help.test.vim:Test_signatures_TopLine\(\)"
echo " - run all tests in a clean env: $0 --basedir \$(mktemp -d) --install"
exit 0
;;
*)
break
;;
esac
done
# Version specific flags, which exist in vim but not nvim or other way around
# vim: --not-a-term: silences a warning
IS_NVIM=$($VIM_EXE --version | grep -cm1 NVIM)
VERSION_OPTIONS=$([ "$IS_NVIM" -ne 0 ] && echo '--headless' || echo '--not-a-term')
RUN_VIM="${VIM_EXE} -N --clean ${VERSION_OPTIONS}"
if which timeout >/dev/null 2>&1; then
TIMEOUT_SCRIPT="timeout --verbose --foreground --kill-after 7m 5m"
else
TIMEOUT_SCRIPT=""
echo "WARNING: No timeout found, not using timeout"
fi
# Check that vim's python support actual works, else we can't run tests
if ! ${TIMEOUT_SCRIPT} ${RUN_VIM} --cmd "py3 import vim; vim.command( 'qa!' )" --cmd "cquit!"; then
echo "Vim's python support doesn't work. Cannot run tests" >&2
exit 1
else
echo "Vim's python support works. Let's go..."
fi
# There seems to be a race condition with the way asciinema sets the pty size.
# It does fork() then if is_child: execvpe(...) else: set_pty_size(). This means
# the child might read the pty size before it's set. So we use a 1s sleepy.
RUN_TEST="sleep 1 && ${TIMEOUT_SCRIPT} ${RUN_VIM} -S lib/run_test.vim"
# We use fd 3 for vim's output. Send it to stdout if not already redirected
# Note: can't use ${out_fd} in a redirect because redirects happen before
# variable substitution
if [ "${out_fd}" = "1" ]; then
exec 3>&1
fi
# Basic pre-flight check that vim and python work
# ffs neovim https://github.com/neovim/neovim/issues/14438
if ${TIMEOUT_SCRIPT} ${RUN_VIM} -u support/test_python3.vim .fail; then
rm -f .fail
echo "$VIM_EXE appears to have working python3 support. Let's go..."
else
cat .fail
rm -f .fail
echo ""
echo ""
echo "ERROR: $VIM_EXE does not appear to have working python3 support."
echo "ERROR: Cannot continue"
exit 1
fi
if [ -z "$VIMSPECTOR_MIMODE" ]; then
case $(uname) in
Linux)
export VIMSPECTOR_MIMODE=gdb
;;
Darwin)
export VIMSPECTOR_MIMODE=lldb
;;
*)
if which lldb-mi >/dev/null 2>&1; then
export VIMSPECTOR_MIMODE=lldb
elif which gdb >/dev/null 2>&1; then
export VIMSPECTOR_MIMODE=gdb
else
echo "Couldn't guess VIMSPECTOR_MIMODE. Specify --gdb/--lldb"
exit 1
fi
;;
esac
fi
if [ -z "$VIMSPECTOR_ARCH" ]; then
export VIMSPECTOR_ARCH=$(uname -m)
if [ "$VIMSPECTOR_ARCH" = "aarch64" ]; then
export VIMSPECTOR_ARCH=arm64
fi
fi
BASEDIR_CMD="let g:vimspector_base_dir='${BASEDIR}'"
TESTDIR=${HOMEDIR}/tests
echo "Testing with:"
echo " * VIMSPECTOR_MIMODE=$VIMSPECTOR_MIMODE"
echo " * VIMSPECTOR_ARCH=$VIMSPECTOR_ARCH"
echo " * RUN_VIM=$RUN_VIM"
echo " * RUN_TEST=$RUN_TEST"
echo " * BASEDIR=$BASEDIR"
echo " * BASEDIR_CMD=$BASEDIR_CMD"
echo " * TESTDIR=$TESTDIR"
if [ "$INSTALL" = "1" ] || [ "$INSTALL" = "script" ]; then
if ! python3 ${HOMEDIR}/install_gadget.py \
--basedir ${BASEDIR} \
${INSTALLER_ARGS} \
--all \
--force-enable-csharp \
--force-enable-node; then
echo "Script installation reported errors" >&2
exit 1
fi
fi
QUIT_ON_FAIL=(-c 'autocmd User VimspectorInstallFailed cquit!')
if [ "$NO_QUIT_ON_FAIL" -eq 1 ]; then
QUIT_ON_FAIL=()
fi
if [ "$INSTALL" = "1" ] || [ "$INSTALL" = "vim" ]; then
if ! $RUN_VIM -u ${HOMEDIR}/tests/vimrc \
--cmd "${BASEDIR_CMD}" \
-c 'autocmd User VimspectorInstallSuccess qa!' \
"${QUIT_ON_FAIL[@]}" \
-c "VimspectorInstall --all netcoredbg vscode-js-debug"; then
echo "Vim installation reported errors" >&2
exit 1
fi
fi
if [ "$UPDATE" = "1" ]; then
if ! $RUN_VIM -u ${HOMEDIR}/tests/vimrc \
--cmd "${BASEDIR_CMD}" \
-c 'autocmd User VimspectorInstallSuccess qa!' \
"${QUIT_ON_FAIL[@]}" \
-c "VimspectorUpdate"; then
echo "Vim update reported errors" >&2
exit 1
fi
fi
if [ "$SKIP_BUILD" = "1" ]; then
echo "%SETUP - Skipped building test programs"
else
echo "%SETUP - Building test programs..."
set -e
pushd tests/testdata/cpp/simple
make clean
make -j all
popd
pushd support/test/csharp
dotnet build
popd
set +e
echo "%DONE - built test programs"
fi
# Start
echo "Running Vimspector Vim tests"
RESULT=0
TESTS="$@"
if [ -z "$TESTS" ]; then
TESTS=${TESTDIR}/*.test.vim
fi
while true; do
for t in ${TESTS}; do
# split on : into fileName and testName
IFS=: read -s tt T <<< "$t"
# Resolve into absolute path ($tt) and test file name ($t)
if [ -f "$tt" ]; then
tt=$(readlink -f "$tt")
t=$(basename "$tt")
elif [ -f "$TESTDIR/$tt" ]; then
t=$tt
tt=$TESTDIR/$t
else
echo "%ERROR: Unable to find a test named $tt ($t) in $TESTDIR"
RESULT=1
break
fi
echo ""
echo "%RUN: $t ($tt)"
TESTLOGDIR=${BASEDIR}/tests/logs/$t
pushd ${TESTDIR} > /dev/null
RUN="${RUN_TEST} --cmd \"${BASEDIR_CMD}\" \
--cmd 'au SwapExists * let v:swapchoice = \"e\"'\
'$tt' '$T'"
if ${ASCIINEMA} rec -q --cols $SCREENCOLS --rows $SCREENROWS --overwrite --command "${RUN}" $t.cast >&3 && [ -f $t.res ]; then
echo "%PASS: $t PASSED"
else
echo "%FAIL: $t FAILED - see $TESTLOGDIR"
RESULT=1
fi
rm -rf $TESTLOGDIR
mkdir -p $TESTLOGDIR
${RUN_VIM} --version > ${TESTLOGDIR}/vimversion
if [ "$VIMSPECTOR_TEST_STDOUT" = "messages" ]; then
if [ -f messages ]; then
echo "%MESSAGES"
cat messages
echo "%END"
else
echo "%MESSAGES"
echo "No messages found"
echo "%END"
fi
fi
mv $t.cast $TESTLOGDIR
for l in messages debuglog test.log *.testlog; do
# In CI we can't view the output files, so we just have to cat them
if [ -f $l ]; then
if [ "$VIMSPECTOR_TEST_STDOUT" = "all" ]; then
echo ""
echo ""
echo "*** START: $l ***"
cat $l
echo "*** END: $l ***"
fi
mv $l $TESTLOGDIR
fi
done
rm -f $t.res
popd > /dev/null
done
if [ $TEST_UNTIL_FAILURE -eq 1 ] && [ $RESULT -eq 0 ]; then
echo "All tests passed. Running again."
continue
fi
break
done
# close out_fd if it's not stdout/stderr/
(( out_fd > 2 )) && exec 3>&-
echo "Done running tests"
echo ""
echo "All done. Exit with ${RESULT}"
exit $RESULT