Add distribution tests

.ci/make-dist.sh

@@ -43,7 +43,8 @@ case "$buildPlatform" in
         filename="qb64_lnx.tar.gz"
         format=tar
 
-        cp -p ./qb64         $DIST_ROOT
+        # Not sure if we should distribute this
+        # cp -p ./qb64         $DIST_ROOT
         cp -p ./setup_lnx.sh $DIST_ROOT
         ;;
 

.github/workflows/ci.yml

@@ -62,7 +62,15 @@ jobs:
       shell: bash
       run: .ci/make-dist.sh ${{ matrix.os }}
 
+      # Note that compiling programs in dist does modify the ./dist and make it
+      # dirty, but that's ok because we've already create the .7z or .tar.gz
+      # artifacts in the previous step
+    - name: Test QB64 Artifact
+      shell: bash
+      run: tests/run_dist_tests.sh ./dist/qb64 ${{ matrix.prefix }}
+
     - uses: actions/upload-artifact@v3
+      if: always()
       with:
         name: qb64-${{ matrix.prefix }}-${{ matrix.platform }}
         path: |

tests/.gitignore

@@ -0,0 +1 @@
+!*.bas

tests/assert.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+TEST_SCRIPT=$1
+shift
+
+. ./tests/colors.sh
+
+export TOTAL_RESULT=0
+export PREFIX=""
+export TESTCASE=""
+export TEST_COUNT=0
+
+# Asserts that the exit code of the previous command was successful (IE. zero)
+#
+# Argument 1: Name for this specific test
+# Argument 2: Optional message to display on error
+# Arguments 3+: Optional Command + arguments to run on error
+assert_success_named ()
+{
+    RESULT=$?
+    NAME=$1
+    ERRORMSG=$2
+    shift
+    shift
+
+    TEST_COUNT=$(($TEST_COUNT + 1))
+
+    printf "$PREFIX: $TEST_COUNT:"
+
+    if ! [ -z "$TESTCASE" ]; then
+        printf " $TESTCASE:"
+    fi
+
+    if ! [ -z "$NAME" ]; then
+        printf " $NAME:"
+    fi
+
+    if [ "$RESULT" -eq 0 ]; then
+        echo "$GREEN PASS!$RESET"
+    else
+        printf "$RED FAILURE!$RESET"
+
+        if ! [ -z "$ERRORMSG" ]; then
+            echo " - $ERRORMSG"
+        else
+            echo
+        fi
+
+        TOTAL_RESULT=$(($TOTAL_RESULT + 1))
+
+        if [ $# -gt 0 ]; then
+            "$@"
+        fi
+    fi
+}
+
+# Asserts that the exit code of the previous command was successful (IE. zero)
+#
+# Argument 1: Optional message to display on error
+# Arguments 2+: Optional Command + arguments to run on error
+assert_success ()
+{
+    assert_success_named "" "$@"
+}
+
+assert_ignored ()
+{
+    TEST_COUNT=$(($TEST_COUNT + 1))
+    echo "$PREFIX: $TEST_COUNT: $TESTCASE:$YELLOW IGNORED!$RESET"
+}
+
+. "$TEST_SCRIPT" "$@"
+
+exit $TOTAL_RESULT

tests/colors.sh

@@ -0,0 +1,5 @@
+
+GREEN=""
+RED=""
+YELLOW=""
+RESET=""

tests/dist/console.bas

@@ -0,0 +1,5 @@
+$CONSOLE:ONLY
+_CONSOLE ON
+
+Print "This is a Dist test"
+SYSTEM

tests/dist/console.result

@@ -0,0 +1 @@
+This is a Dist test

tests/dist_tests.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Arg 1: Dist location
+
+PREFIX="dist"
+
+# Get absolute paths for everything since we're going to cd into the distribution directory
+ROOT=$(pwd)
+TEST_CASES="$ROOT/tests/dist"
+
+RESULTS_DIR="$ROOT/tests/results/$PREFIX"
+OUTPUT_DIR="$ROOT/tests/output/$PREFIX"
+
+mkdir -p "$RESULTS_DIR"
+mkdir -p "$OUTPUT_DIR"
+
+# Move into distribution location
+cd $1
+
+# Specific steps for each platform
+case "$2" in
+    win)
+        ;;
+
+    lnx)
+        ./setup_lnx.sh 1>"$RESULTS_DIR/linux-setup.txt"
+        assert_success_named "Linux setup" cat "$RESULTS_DIR/linux-setup.txt"
+        ;;
+
+    osx)
+        ./setup_osx.command 1>"$RESULTS_DIR/osx-setup.txt"
+        assert_success_named "OSX setup" cat "$RESULTS_DIR/osx-setup.txt"
+        ;;
+esac
+
+show_failure()
+{
+    cat "$RESULTS_DIR/$1-compile_result.txt"
+    cat "$RESULTS_DIR/$1-compilelog.txt"
+}
+
+for basFile in $TEST_CASES/*.bas
+do 
+    test=$(basename $basFile .bas)
+    outputExe="$OUTPUT_DIR/$test-output"
+
+    TESTCASE=$test
+
+    ./qb64 -x  "$TEST_CASES/$test.bas" -o "$outputExe" 1>$RESULTS_DIR/$test-compile_result.txt
+    ERR=$?
+    cp ./internal/temp/compilelog.txt $RESULTS_DIR/$test-compilelog.txt
+
+    (exit $ERR)
+    assert_success_named "compile" "Compilation Error:" show_failure $test
+
+    test -f "$outputExe"
+    assert_success_named "exe exists" "output.exe does not exist!" show_failure $test
+
+    testResult=$("$outputExe" 2>&1)
+    assert_success_named "run" "Execution Error:" echo "$testResult"
+
+    [ "$testResult" == "$(cat $TEST_CASES/$test.result)" ]
+    assert_success_named "result" "Result is wrong:" echo "$testResult"
+done

tests/run_dist_tests.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+result=0
+
+./tests/assert.sh ./tests/dist_tests.sh "$1" "$2" || result=1
+
+exit $result