From e391ef5631cdc2a9f7f69504cd1e57d7ca510969 Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Mon, 29 Jul 2024 12:30:30 +0200 Subject: [PATCH 1/2] main: prevent invalid memory access when executing empty stdin In case the ucode cli executes stdin with zero bytes length, ensure to pass a dummy string instead of a NULL pointer to uc_source_new_buffer() to prevent libc's fmemopen() from writing to nonexistent memory. Signed-off-by: Jo-Philipp Wich --- main.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main.c b/main.c index a51c3707..96f4adb9 100644 --- a/main.c +++ b/main.c @@ -201,6 +201,14 @@ read_stdin(void) stdin_unused = NULL; + /* On empty stdin, provide a dummy buffer and ensure that it is + * at least one byte long, due to + * https://github.com/google/sanitizers/issues/627 */ + if (p == NULL) { + p = xstrdup("\n"); + tlen = 1; + } + return uc_source_new_buffer("[stdin]", p, tlen); } From fbabec42349880407c4308211129c07ff51c484a Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Mon, 29 Jul 2024 12:32:56 +0200 Subject: [PATCH 2/2] tests: replace test runner shell script with ucode implementation The ucode interpreter and libraries are mature enough to execute their own testcases now, so replace the existing shell script with an equivalent ucode implementation. Signed-off-by: Jo-Philipp Wich --- tests/custom/CMakeLists.txt | 4 +- tests/custom/run_tests.sh | 240 ---------------------------------- tests/custom/run_tests.uc | 254 ++++++++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 242 deletions(-) delete mode 100755 tests/custom/run_tests.sh create mode 100755 tests/custom/run_tests.uc diff --git a/tests/custom/CMakeLists.txt b/tests/custom/CMakeLists.txt index c8007a0e..c94278e0 100644 --- a/tests/custom/CMakeLists.txt +++ b/tests/custom/CMakeLists.txt @@ -1,6 +1,6 @@ ADD_TEST( NAME custom - COMMAND run_tests.sh + COMMAND $ -L $/*.so -S run_tests.uc WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT @@ -11,7 +11,7 @@ SET_PROPERTY(TEST custom APPEND PROPERTY ENVIRONMENT IF(CMAKE_C_COMPILER_ID STREQUAL "Clang") ADD_TEST( NAME custom-san - COMMAND run_tests.sh + COMMAND $ -L $/*.so -S run_tests.uc WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/tests/custom/run_tests.sh b/tests/custom/run_tests.sh deleted file mode 100755 index 96ac783e..00000000 --- a/tests/custom/run_tests.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env bash - -if greadlink -f . &>/dev/null; then - readlink=greadlink -else - readlink=readlink -fi - -testdir=$(dirname "$0") -topdir=$($readlink -f "$testdir/../..") - -line='........................................' -ucode_bin=${UCODE_BIN:-"$topdir/ucode"} -ucode_lib=${UCODE_LIB:-"$topdir"} - -extract_sections() { - local file=$1 - local dir=$2 - local count=0 - local tag line outfile - - while IFS= read -r line; do - case "$line" in - "-- Args --") - tag="args" - count=$((count + 1)) - outfile=$(printf "%s/%03d.args" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Vars --") - tag="vars" - count=$((count + 1)) - outfile=$(printf "%s/%03d.vars" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Testcase --") - tag="test" - count=$((count + 1)) - outfile=$(printf "%s/%03d.in" "$dir" $count) - printf "" > "$outfile" - ;; - "-- Expect stdout --"|"-- Expect stderr --"|"-- Expect exitcode --") - tag="${line#-- Expect }" - tag="${tag% --}" - count=$((count + 1)) - outfile=$(printf "%s/%03d.%s" "$dir" $count "$tag") - printf "" > "$outfile" - ;; - "-- File "*" --") - tag="file" - outfile="${line#-- File }" - outfile="$(echo "${outfile% --}" | xargs)" - outfile="$dir/files$($readlink -m "/${outfile:-file}")" - mkdir -p "$(dirname "$outfile")" - printf "" > "$outfile" - ;; - "-- End (no-eol) --") - truncate -s -1 "$outfile" - tag="" - outfile="" - ;; - "-- End --") - tag="" - outfile="" - ;; - *) - if [ -n "$tag" ]; then - printf "%s\\n" "$line" >> "$outfile" - fi - ;; - esac - done < "$file" - - return $(ls -l "$dir/"*.in 2>/dev/null | wc -l) -} - -run_testcase() { - local num=$1 - local dir=$2 - local in=$3 - local out=$4 - local err=$5 - local code=$6 - local args=$7 - local vars=$8 - local fail=0 - - ( - cd "$dir" - - IFS=$'\n' - - local var - for var in $vars; do - case "$var" in - *=*) export "$var" ;; - esac - done - - IFS=$' \t\n' - - $ucode_bin -T"," -L "$ucode_lib/*.so" -D TESTFILES_PATH="$($readlink -f "$dir/files")" $args - <"$in" >"$dir/res.out" 2>"$dir/res.err" - ) - - printf "%d\n" $? > "$dir/res.code" - touch "$dir/empty" - - sed -i -e "s#$dir#.#g" "$dir/res.out" "$dir/res.err" - - if ! cmp -s "$dir/res.err" "${err:-$dir/empty}"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected stderr did not match:\n" $num - diff -au --color=always --label="Expected stderr" --label="Resulting stderr" "${err:-$dir/empty}" "$dir/res.err" - printf -- "---\n" - fail=1 - fi - - if ! cmp -s "$dir/res.out" "${out:-$dir/empty}"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected stdout did not match:\n" $num - diff -au --color=always --label="Expected stdout" --label="Resulting stdout" "${out:-$dir/empty}" "$dir/res.out" - printf -- "---\n" - fail=1 - fi - - if [ -n "$code" ] && ! cmp -s "$dir/res.code" "$code"; then - [ $fail = 0 ] && printf "!\n" - printf "Testcase #%d: Expected exit code did not match:\n" $num - diff -au --color=always --label="Expected code" --label="Resulting code" "$code" "$dir/res.code" - printf -- "---\n" - fail=1 - fi - - return $fail -} - -run_test() { - local file=$1 - local name=${file##*/} - local res ecode eout eerr ein eargs tests - local testcase_first=0 failed=0 count=0 - - printf "%s %s " "$name" "${line:${#name}}" - - mkdir "/tmp/test.$$" - - extract_sections "$file" "/tmp/test.$$" - tests=$? - - [ -f "/tmp/test.$$/001.in" ] && testcase_first=1 - - for res in "/tmp/test.$$/"[0-9]*; do - case "$res" in - *.in) - count=$((count + 1)) - - if [ $testcase_first = 1 ]; then - # Flush previous test - if [ -n "$ein" ]; then - run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - eout="" - eerr="" - ecode="" - eargs="" - evars="" - fi - - ein=$res - else - run_testcase $count "/tmp/test.$$" "$res" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - - eout="" - eerr="" - ecode="" - eargs="" - evars="" - fi - - ;; - *.stdout) eout=$res ;; - *.stderr) eerr=$res ;; - *.exitcode) ecode=$res ;; - *.args) eargs=$(cat "$res") ;; - *.vars) evars=$(cat "$res") ;; - esac - done - - # Flush last test - if [ $testcase_first = 1 ] && [ -n "$eout$eerr$ecode" ]; then - run_testcase $count "/tmp/test.$$" "$ein" "$eout" "$eerr" "$ecode" "$eargs" "$evars" || failed=$((failed + 1)) - fi - - rm -r "/tmp/test.$$" - - if [ $failed = 0 ]; then - printf "OK\n" - else - printf "%s %s FAILED (%d/%d)\n" "$name" "${line:${#name}}" $failed $tests - fi - - return $failed -} - - -n_tests=0 -n_fails=0 - -select_tests="$@" - -use_test() { - local input="$($readlink -f "$1")" - local test - - [ -f "$input" ] || return 1 - [ -n "$select_tests" ] || return 0 - - for test in "$select_tests"; do - test="$($readlink -f "$test")" - - [ "$test" != "$input" ] || return 0 - done - - return 1 -} - -for catdir in "$testdir/"[0-9][0-9]_*; do - [ -d "$catdir" ] || continue - - printf "\n##\n## Running %s tests\n##\n\n" "${catdir##*/[0-9][0-9]_}" - - for testfile in "$catdir/"[0-9][0-9]_*; do - use_test "$testfile" || continue - - n_tests=$((n_tests + 1)) - run_test "$testfile" || n_fails=$((n_fails + 1)) - done -done - -printf "\nRan %d tests, %d okay, %d failures\n" $n_tests $((n_tests - n_fails)) $n_fails -exit $n_fails diff --git a/tests/custom/run_tests.uc b/tests/custom/run_tests.uc new file mode 100755 index 00000000..ff81afbf --- /dev/null +++ b/tests/custom/run_tests.uc @@ -0,0 +1,254 @@ +#!/usr/bin/env -S ucode -S + +import * as fs from 'fs'; + +let testdir = sourcepath(0, true); +let topdir = fs.realpath(`${testdir}/../..`); + +let line = '........................................'; +let ucode_bin = getenv('UCODE_BIN') || `${topdir}/ucode`; +let ucode_lib = getenv('UCODE_LIB') || topdir; + +function mkdir_p(path) { + let parts = split(rtrim(path, '/') || '/', /\/+/); + let current = ''; + + for (let part in parts) { + current += part + '/'; + + let s = fs.stat(current); + + if (s == null) { + if (!fs.mkdir(current)) + die(`Error creating directory '${current}': ${fs.error()}`); + } + else if (s.type != 'directory') { + die(`Path '${current}' exists but is not a directory`); + } + } +} + +function shellquote(s) { + return `'${replace(s, "'", "'\\''")}'`; +} + +function getpid() { + return +fs.popen('echo $PPID', 'r').read('all'); +} + +function has_expectations(testcase) +{ + return (testcase?.stdout != null || testcase?.stderr != null || testcase?.exitcode != null); +} + +function parse_testcases(file, dir) { + let fp = fs.open(file, 'r') ?? die(`Unable to open ${file}: ${fs.error()}`); + let testcases, testcase, section, m; + let code_first = false; + + for (let line = fp.read('line'); length(line); line = fp.read('line')) { + if (line == '-- Args --\n') { + section = [ 'args', [] ]; + } + else if (line == '-- Vars --\n') { + section = [ 'env', {} ]; + } + else if (line == '-- Testcase --\n') { + section = [ 'code', '' ]; + } + else if ((m = match(line, /^-- Expect (stdout|stderr|exitcode) --$/s)) != null) { + section = [ m[1], '' ]; + } + else if ((m = match(line, /^-- File (.*)--$/s)) != null) { + section = [ 'file', `${dir}/files/${trim(m[1]) || 'file'}`, '' ]; + } + else if ((m = match(line, /^-- End( \(no-eol\))? --$/s)) != null) { + if (m[1] != null && type(section[-1]) == 'string') + section[-1] = substr(section[-1], 0, -1); + + if (section[0] == 'code') { + if (testcases == null && !has_expectations(testcase)) + code_first = true; + + if (code_first) { + if (testcase?.code != null) { + push(testcases ??= [], testcase); + testcase = null; + } + + (testcase ??= {}).code = section[1]; + } + else { + push(testcases ??= [], { ...testcase, code: section[1] }); + testcase = null; + } + } + else if (section[0] == 'file') { + ((testcase ??= {}).files ??= {})[section[1]] = section[2]; + } + else { + (testcase ??= {})[section[0]] = section[1]; + } + + section = null; + } + else if (section) { + switch (section[0]) { + case 'args': + if ((m = trim(line)) != '') + push(section[1], ...split(m, /[ \t\r\n]+/)); + break; + + case 'env': + if ((m = match(line, /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/s)) != null) + section[1][m[1]] = m[2]; + break; + + default: + section[-1] += line; + break; + } + } + } + + if (code_first && testcase.code != null && has_expectations(testcase)) + push(testcases ??= [], testcase); + + return testcases; +} + +function diff(tag, ...ab) { + let cmd = [ 'diff', '-au', '--color=always', `--label=Expected ${tag}`, `--label=Resulting ${tag}` ]; + let tmpfiles = []; + + for (let i, f in ab) { + if (type(f) != 'resource') { + push(tmpfiles, fs.mkstemp()); + tmpfiles[-1].write(f); + f = tmpfiles[-1]; + } + + f.seek(0); + push(cmd, `/dev/fd/${f.fileno()}`); + } + + system(cmd); +} + +function run_testcase(num, dir, testcase) { + let fout = fs.mkstemp(`${dir}/stdout.XXXXXX`); + let ferr = fs.mkstemp(`${dir}/stderr.XXXXXX`); + + let eout = testcase.stdout ?? ''; + let eerr = testcase.stderr ?? ''; + let ecode = testcase.exitcode ? +testcase.exitcode : null; + + let cmd = join(' ', [ + ...map(keys(testcase.env) ?? [], k => `export ${k}=${shellquote(testcase.env[k])};`), + `cd ${shellquote(dir)};`, + `exec ${ucode_bin}`, + `-T','`, + `-L ${shellquote(`${ucode_lib}/*.so`)}`, + `-D TESTFILES_PATH=${shellquote(`${fs.realpath(dir)}/files`)}`, + `${join(' ', map(testcase.args ?? [], shellquote))} -`, + `>/dev/fd/${fout.fileno()} 2>/dev/fd/${ferr.fileno()}` + ]); + + let proc = fs.popen(cmd, 'w') ?? die(`Error launching test command "${cmd}": ${fs.error()}\n`); + + if (testcase.code != null) + proc.write(testcase.code); + + let exitcode = proc.close(); + + fout.seek(0); + ferr.seek(0); + + let ok = true; + + if (replace(ferr.read('all'), dir, '.') != eerr) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected stderr did not match:\n", num); + diff('stderr', eerr, ferr); + print("---\n"); + ok = false; + } + + if (replace(fout.read('all'), dir, '.') != eout) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected stdout did not match:\n", num); + diff('stdout', eout, fout); + print("---\n"); + ok = false; + } + + if (ecode != null && exitcode != ecode) { + if (ok) print('!\n'); + printf("Testcase #%d: Expected exit code did not match:\n", num); + diff('code', `${ecode}\n`, `${exitcode}\n`); + print("---\n"); + ok = false; + } + + return ok; +} + +function run_test(file) { + let name = fs.basename(file); + printf('%s %s ', name, substr(line, length(name))); + + let tmpdir = sprintf('/tmp/test.%d', getpid()); + let testcases = parse_testcases(file, tmpdir); + let failed = 0; + + fs.mkdir(tmpdir); + + try { + for (let i, testcase in testcases) { + for (let path, data in testcase.files) { + mkdir_p(fs.dirname(path)); + fs.writefile(path, data) ?? die(`Error writing testcase file "${path}": ${fs.error()}\n`); + } + + failed += !run_testcase(i + 1, tmpdir, testcase); + } + } + catch (e) { + warn(`${e.type}: ${e.message}\n${e.stacktrace[0].context}\n`); + } + + system(['rm', '-r', tmpdir]); + + if (failed == 0) + print('OK\n'); + else + printf('%s %s FAILED (%d/%d)\n', name, substr(line, length(name)), failed, length(testcases)); + + return failed; +} + +let n_tests = 0; +let n_fails = 0; +let select_tests = filter(map(ARGV, p => fs.realpath(p)), length); + +function use_test(input) { + return fs.access(input = fs.realpath(input)) && + (!length(select_tests) || filter(select_tests, p => p == input)[0]); +} + +for (let catdir in fs.glob(`${testdir}/[0-9][0-9]_*`)) { + if (fs.stat(catdir)?.type != 'directory') + continue; + + printf('\n##\n## Running %s tests\n##\n\n', substr(fs.basename(catdir), 3)); + + for (let testfile in fs.glob(`${catdir}/[0-9][0-9]_*`)) { + if (!use_test(testfile)) continue; + + n_tests++; + n_fails += run_test(testfile); + } +} + +printf('\nRan %d tests, %d okay, %d failures\n', n_tests, n_tests - n_fails, n_fails); +exit(n_fails);