From 65db95c87678856ae0f164fd093f5d960755d2e1 Mon Sep 17 00:00:00 2001
From: Matthew Kilgore <mattkilgore12@gmail.com>
Date: Sat, 3 Sep 2022 20:55:10 -0400
Subject: [PATCH] Add support for error tests

This adds support for tests of the error output of QB64-PE. The tests
are in the form of a `.bas` file that does not compile, and a `.err`
file containing the expected output of QB64-PE when producing an error
for that file.

Additionally the `compile_tests` now support multiple tests in a single
folder, via using multiple `.bas` files with different names.
---
 tests/compile_tests.sh | 94 ++++++++++++++++++++++++++++--------------
 1 file changed, 64 insertions(+), 30 deletions(-)

diff --git a/tests/compile_tests.sh b/tests/compile_tests.sh
index bce4bfee5..66cbedd53 100755
--- a/tests/compile_tests.sh
+++ b/tests/compile_tests.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 # Arg 1: qb54 location
+# Arg 2: Optional category to test
 
 PREFIX="Compilation"
 
@@ -9,10 +10,14 @@ mkdir -p $RESULTS_DIR
 
 QB64=$1
 
+if [ "$#" -eq 2 ]; then
+    CATEGORY="/$2"
+fi
+
 show_failure()
 {
-    cat "$RESULTS_DIR/$1-compile_result.txt"
-    cat "$RESULTS_DIR/$1-compilelog.txt"
+    cat "$RESULTS_DIR/$1-$2-compile_result.txt"
+    cat "$RESULTS_DIR/$2-compilelog.txt"
 }
 
 show_incorrect_result()
@@ -21,41 +26,70 @@ show_incorrect_result()
     printf "GOT:      '%s'\n" "$2"
 }
 
-for test in ./tests/compile_tests/*
-do 
-    test=$(basename "$test")
 
-    TESTCASE="$test"
-    EXE="$RESULTS_DIR/$test - output"
+# Each .bas file represents a separate test.
+while IFS= read -r test
+do 
+    category=$(basename "$(dirname "$test")")
+    testName=$(basename "$test" .bas)
+
+    TESTCASE="$category/$testName"
+    EXE="$RESULTS_DIR/$category-$testName - output"
+
+    # If a .err file exists, then this test is actually testing a compilation error
+    testType="success"
+    if test -f "./tests/compile_tests/$category/$testName.err"; then
+        testType="error"
+    fi
 
     # Clear out temp folder before next compile, avoids stale compilelog files
     rm -fr ./internal/temp/*
 
-    "$QB64" -x  "./tests/compile_tests/$test/test.bas" -o "$EXE" 1>"$RESULTS_DIR/$test-compile_result.txt"
+    # Clean up existing EXE, so we don't use it by accident
+    rm -f "$EXE*"
+
+    compileResultOutput="$RESULTS_DIR/$category-$testName-compile_result.txt"
+
+    "$QB64" -q -x "./tests/compile_tests/$category/$testName.bas" -o "$EXE" 1>"$compileResultOutput"
     ERR=$?
-    cp_if_exists ./internal/temp/compilelog.txt "$RESULTS_DIR/$test-compilelog.txt"
+    cp_if_exists ./internal/temp/compilelog.txt "$RESULTS_DIR/$category-$testName-compilelog.txt"
 
-    (exit $ERR)
-    assert_success_named "Compile" "Compilation Error:" show_failure "$test"
+    if [ "$testType" == "success" ]; then
+        (exit $ERR)
+        assert_success_named "Compile" "Compilation Error:" show_failure "$category" "$testName"
 
-    test -f "$EXE"
-    assert_success_named "exe exists" "$test-output executable does not exist!" show_failure "$test"
+        test -f "$EXE"
+        assert_success_named "exe exists" "$test-output executable does not exist!" show_failure "$category" "$testName"
 
-    if [ ! -f "./tests/compile_tests/$test/test.output" ]; then
-        continue
+        # Some tests do not have an output or err file because they should
+        # compile successfully but cannot be run on the build agents
+        if [ ! -f "./tests/compile_tests/$category/$testName.output" ]; then
+            continue
+        fi
+
+        expectedResult="$(cat "./tests/compile_tests/$category/$testName.output")"
+
+        pushd . > /dev/null
+        cd "./tests/compile_tests/$category"
+        testResult=$("../../../$EXE" 2>&1)
+        ERR=$?
+        popd > /dev/null
+
+        (exit $ERR)
+        assert_success_named "run" "Execution Error:" echo "$testResult"
+
+        [ "$testResult" == "$expectedResult" ]
+        assert_success_named "result" "Result is wrong:" show_incorrect_result "$expectedResult" "$testResult"
+    else
+        ! (exit $ERR)
+        assert_success_named "Compile" "Compilation Success, was expecting error:" show_failure "$category" "$testName"
+
+        ! test -f "$EXE"
+        assert_success_named "Exe exists" "'$category-$testName - output' exists, it should not!" show_failure "$category" "$testName"
+
+        expectedErr="$(cat "./tests/compile_tests/$category/$testName.err")"
+
+        diffResult=$(diff -y "./tests/compile_tests/$category/$testName.err" "$compileResultOutput")
+        assert_success_named "Error result" "Error reporting is wrong:" echo "$diffResult"
     fi
-
-    expectedResult="$(cat "./tests/compile_tests/$test/test.output")"
-
-    pushd . > /dev/null
-    cd "./tests/compile_tests/$test"
-    testResult=$("../../../$EXE" 2>&1)
-    ERR=$?
-    popd > /dev/null
-
-    (exit $ERR)
-    assert_success_named "run" "Execution Error:" echo "$testResult"
-
-    [ "$testResult" == "$expectedResult" ]
-    assert_success_named "result" "Result is wrong:" show_incorrect_result "$expectedResult" "$testResult"
-done
+done < <(find ./tests/compile_tests$CATEGORY -name "*.bas" -print)