mirror of
https://github.com/QB64-Phoenix-Edition/QB64pe.git
synced 2024-06-29 12:50:37 +00:00
Add support for error tests
This adds support for tests of the error output of QB64-PE. The tests are in the form of a `.bas` file that does not compile, and a `.err` file containing the expected output of QB64-PE when producing an error for that file. Additionally the `compile_tests` now support multiple tests in a single folder, via using multiple `.bas` files with different names.
This commit is contained in:
parent
64d86326bf
commit
65db95c876
|
@ -1,5 +1,6 @@
|
|||
#!/bin/bash
|
||||
# Arg 1: qb54 location
|
||||
# Arg 2: Optional category to test
|
||||
|
||||
PREFIX="Compilation"
|
||||
|
||||
|
@ -9,10 +10,14 @@ mkdir -p $RESULTS_DIR
|
|||
|
||||
QB64=$1
|
||||
|
||||
if [ "$#" -eq 2 ]; then
|
||||
CATEGORY="/$2"
|
||||
fi
|
||||
|
||||
show_failure()
|
||||
{
|
||||
cat "$RESULTS_DIR/$1-compile_result.txt"
|
||||
cat "$RESULTS_DIR/$1-compilelog.txt"
|
||||
cat "$RESULTS_DIR/$1-$2-compile_result.txt"
|
||||
cat "$RESULTS_DIR/$2-compilelog.txt"
|
||||
}
|
||||
|
||||
show_incorrect_result()
|
||||
|
@ -21,41 +26,70 @@ show_incorrect_result()
|
|||
printf "GOT: '%s'\n" "$2"
|
||||
}
|
||||
|
||||
for test in ./tests/compile_tests/*
|
||||
do
|
||||
test=$(basename "$test")
|
||||
|
||||
TESTCASE="$test"
|
||||
EXE="$RESULTS_DIR/$test - output"
|
||||
# Each .bas file represents a separate test.
|
||||
while IFS= read -r test
|
||||
do
|
||||
category=$(basename "$(dirname "$test")")
|
||||
testName=$(basename "$test" .bas)
|
||||
|
||||
TESTCASE="$category/$testName"
|
||||
EXE="$RESULTS_DIR/$category-$testName - output"
|
||||
|
||||
# If a .err file exists, then this test is actually testing a compilation error
|
||||
testType="success"
|
||||
if test -f "./tests/compile_tests/$category/$testName.err"; then
|
||||
testType="error"
|
||||
fi
|
||||
|
||||
# Clear out temp folder before next compile, avoids stale compilelog files
|
||||
rm -fr ./internal/temp/*
|
||||
|
||||
"$QB64" -x "./tests/compile_tests/$test/test.bas" -o "$EXE" 1>"$RESULTS_DIR/$test-compile_result.txt"
|
||||
# Clean up existing EXE, so we don't use it by accident
|
||||
rm -f "$EXE*"
|
||||
|
||||
compileResultOutput="$RESULTS_DIR/$category-$testName-compile_result.txt"
|
||||
|
||||
"$QB64" -q -x "./tests/compile_tests/$category/$testName.bas" -o "$EXE" 1>"$compileResultOutput"
|
||||
ERR=$?
|
||||
cp_if_exists ./internal/temp/compilelog.txt "$RESULTS_DIR/$test-compilelog.txt"
|
||||
cp_if_exists ./internal/temp/compilelog.txt "$RESULTS_DIR/$category-$testName-compilelog.txt"
|
||||
|
||||
(exit $ERR)
|
||||
assert_success_named "Compile" "Compilation Error:" show_failure "$test"
|
||||
if [ "$testType" == "success" ]; then
|
||||
(exit $ERR)
|
||||
assert_success_named "Compile" "Compilation Error:" show_failure "$category" "$testName"
|
||||
|
||||
test -f "$EXE"
|
||||
assert_success_named "exe exists" "$test-output executable does not exist!" show_failure "$test"
|
||||
test -f "$EXE"
|
||||
assert_success_named "exe exists" "$test-output executable does not exist!" show_failure "$category" "$testName"
|
||||
|
||||
if [ ! -f "./tests/compile_tests/$test/test.output" ]; then
|
||||
continue
|
||||
# Some tests do not have an output or err file because they should
|
||||
# compile successfully but cannot be run on the build agents
|
||||
if [ ! -f "./tests/compile_tests/$category/$testName.output" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
expectedResult="$(cat "./tests/compile_tests/$category/$testName.output")"
|
||||
|
||||
pushd . > /dev/null
|
||||
cd "./tests/compile_tests/$category"
|
||||
testResult=$("../../../$EXE" 2>&1)
|
||||
ERR=$?
|
||||
popd > /dev/null
|
||||
|
||||
(exit $ERR)
|
||||
assert_success_named "run" "Execution Error:" echo "$testResult"
|
||||
|
||||
[ "$testResult" == "$expectedResult" ]
|
||||
assert_success_named "result" "Result is wrong:" show_incorrect_result "$expectedResult" "$testResult"
|
||||
else
|
||||
! (exit $ERR)
|
||||
assert_success_named "Compile" "Compilation Success, was expecting error:" show_failure "$category" "$testName"
|
||||
|
||||
! test -f "$EXE"
|
||||
assert_success_named "Exe exists" "'$category-$testName - output' exists, it should not!" show_failure "$category" "$testName"
|
||||
|
||||
expectedErr="$(cat "./tests/compile_tests/$category/$testName.err")"
|
||||
|
||||
diffResult=$(diff -y "./tests/compile_tests/$category/$testName.err" "$compileResultOutput")
|
||||
assert_success_named "Error result" "Error reporting is wrong:" echo "$diffResult"
|
||||
fi
|
||||
|
||||
expectedResult="$(cat "./tests/compile_tests/$test/test.output")"
|
||||
|
||||
pushd . > /dev/null
|
||||
cd "./tests/compile_tests/$test"
|
||||
testResult=$("../../../$EXE" 2>&1)
|
||||
ERR=$?
|
||||
popd > /dev/null
|
||||
|
||||
(exit $ERR)
|
||||
assert_success_named "run" "Execution Error:" echo "$testResult"
|
||||
|
||||
[ "$testResult" == "$expectedResult" ]
|
||||
assert_success_named "result" "Result is wrong:" show_incorrect_result "$expectedResult" "$testResult"
|
||||
done
|
||||
done < <(find ./tests/compile_tests$CATEGORY -name "*.bas" -print)
|
||||
|
|
Loading…
Reference in a new issue