Change how error handling is done in libkleeRuntest.

Previously error messages would be emitted but execution would continue
which might not be desirable.

Now a wrapper function (for fprintf) `report_internal_error()` is used
which will cause the program to exit. The older behaviour of continuing
to execute after an error can be achieved by setting a new environment
variable `KLEE_RUN_TEST_ERRORS_NON_FATAL`.

This commit also adds a test for each error case.

7 files changed, 290 insertions, 13 deletions

diff --git a/runtime/Runtest/intrinsics.c b/runtime/Runtest/intrinsics.c
index d67f093e..4d785ee2 100644
--- a/runtime/Runtest/intrinsics.c
+++ b/runtime/Runtest/intrinsics.c
@@ -10,8 +10,9 @@
 /* Straight C for linking simplicity */
 
 #include <assert.h>
-#include <stdlib.h>
+#include <stdarg.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <sys/mman.h>
 #include <sys/time.h>
@@ -31,6 +32,23 @@ static unsigned char rand_byte(void) {
   return x & 0xFF;
 }
 
+static void report_internal_error(const char *msg, ...)
+    __attribute__((format(printf, 1, 2)));
+static void report_internal_error(const char *msg, ...) {
+  fprintf(stderr, "KLEE_RUN_TEST_ERROR: ");
+  va_list ap;
+  va_start(ap, msg);
+  vfprintf(stderr, msg, ap);
+  va_end(ap);
+  fprintf(stderr, "\n");
+  char *testErrorsNonFatal = getenv("KLEE_RUN_TEST_ERRORS_NON_FATAL");
+  if (testErrorsNonFatal) {
+    fprintf(stderr, "KLEE_RUN_TEST_ERROR: Forcing execution to continue\n");
+  } else {
+    exit(1);
+  }
+}
+
 void klee_make_symbolic(void *array, size_t nbytes, const char *name) {
   static int rand_init = -1;
 
@@ -82,7 +100,7 @@ void klee_make_symbolic(void *array, size_t nbytes, const char *name) {
 
   for (;; ++testPosition) {
     if (testPosition >= testData->numObjects) {
-      fprintf(stderr, "ERROR: out of inputs, using zero\n");
+      report_internal_error("out of inputs. Will use zero if continuing.");
       memset(array, 0, nbytes);
       break;
     } else {
@@ -95,13 +113,14 @@ void klee_make_symbolic(void *array, size_t nbytes, const char *name) {
         continue;
       }
       if (strcmp(name, o->name) != 0) {
-        fprintf(stderr, "ERROR: object name mismatch. Requesting \"%s\" but "
-                        "returning \"%s\"",
-                name, o->name);
+        report_internal_error(
+            "object name mismatch. Requesting \"%s\" but returning \"%s\"",
+            name, o->name);
       }
       memcpy(array, o->bytes, nbytes < o->numBytes ? nbytes : o->numBytes);
       if (nbytes != o->numBytes) {
-        fprintf(stderr, "ERROR: object sizes differ\n");
+        report_internal_error("object sizes differ. Expected %zu but got %u",
+                              nbytes, o->numBytes);
         if (o->numBytes < nbytes)
           memset((char *)array + o->numBytes, 0, nbytes - o->numBytes);
       }
@@ -119,14 +138,13 @@ uintptr_t klee_choose(uintptr_t n) {
   uintptr_t x;
   klee_make_symbolic(&x, sizeof x, "klee_choose");
   if(x >= n)
-    fprintf(stderr, "ERROR: max = %ld, got = %ld\n", n, x);
-  assert(x < n);
+    report_internal_error("klee_choose failure. max = %ld, got = %ld\n", n, x);
   return x;
 }
 
 void klee_assume(uintptr_t x) {
   if (!x) {
-    fprintf(stderr, "ERROR: invalid klee_assume\n");
+    report_internal_error("invalid klee_assume");
   }
 }
 
@@ -148,10 +166,8 @@ int klee_range(int begin, int end, const char* name) {
   int x;
   klee_make_symbolic(&x, sizeof x, name);
   if (x<begin || x>=end) {
-    fprintf(stderr, 
-            "KLEE: ERROR: invalid klee_range(%u,%u,%s) value, got: %u\n", 
-            begin, end, name, x);
-    abort();
+    report_internal_error("invalid klee_range(%u,%u,%s) value, got: %u\n",
+                          begin, end, name, x);
   }
   return x;
 }
diff --git a/test/Replay/libkleeruntest/replay_invalid_klee_assume.c b/test/Replay/libkleeruntest/replay_invalid_klee_assume.c
new file mode 100644
index 00000000..12ac006e
--- /dev/null
+++ b/test/Replay/libkleeruntest/replay_invalid_klee_assume.c
@@ -0,0 +1,44 @@
+// RUN: %llvmgcc -DASSUME_VALUE=1 %s -emit-llvm -g -O0 -c -o %t.bc
+// RUN: rm -rf %t.klee-out
+// RUN: %klee --output-dir=%t.klee-out --search=dfs %t.bc
+// RUN: test -f %t.klee-out/test000001.ktest
+// RUN: test ! -f %t.klee-out/test000002.ktest
+
+// Now try to replay with libkleeRuntest but build the binary to use a different
+// value for the `klee_assume()` call.
+// RUN: %cc -DASSUME_VALUE=32 -DPRINT_VALUE %s %libkleeruntest -Wl,-rpath=%libkleeruntestdir -o %t_runner
+
+// Check that the default is to exit with an error
+// RUN: not env KTEST_FILE=%t.klee-out/test000001.ktest %t_runner 2>&1 | FileCheck -check-prefix=CHECK_FATAL %s
+
+// Check that setting `KLEE_RUN_TEST_ERRORS_NON_FATAL` will not exit with an error
+// and will continue executing.
+// RUN: env KTEST_FILE=%t.klee-out/test000001.ktest KLEE_RUN_TEST_ERRORS_NON_FATAL=1 %t_runner 2>&1 | FileCheck %s
+
+#include "klee/klee.h"
+#include <stdint.h>
+#include <stdio.h>
+
+#ifndef ASSUME_VALUE
+#error ASSUME_VALUE must be defined
+#endif
+
+
+int main(int argc, char** argv) {
+  int x = 54;
+  klee_make_symbolic(&x, sizeof(x), "x");
+  klee_assume(x == ASSUME_VALUE);
+
+#ifdef PRINT_VALUE
+  printf("x=%d\n", x);
+#endif
+
+  return 0;
+}
+// CHECK: KLEE_RUN_TEST_ERROR: invalid klee_assume
+// CHECK: KLEE_RUN_TEST_ERROR: Forcing execution to continue
+// CHECK: x=1
+
+// CHECK_FATAL: KLEE_RUN_TEST_ERROR: invalid klee_assume
+// CHECK_FATAL-NOT: x=1
+
diff --git a/test/Replay/libkleeruntest/replay_invalid_klee_choose.c b/test/Replay/libkleeruntest/replay_invalid_klee_choose.c
new file mode 100644
index 00000000..62f514bf
--- /dev/null
+++ b/test/Replay/libkleeruntest/replay_invalid_klee_choose.c
@@ -0,0 +1,45 @@
+// RUN: %llvmgcc -DBOUND_VALUE=32 -DFORCE_VALUE=20 %s -emit-llvm -g -O0 -c -o %t.bc
+// RUN: rm -rf %t.klee-out
+// RUN: %klee --output-dir=%t.klee-out --libc=klee --search=dfs %t.bc
+// RUN: test -f %t.klee-out/test000001.ktest
+// RUN: test ! -f %t.klee-out/test000002.ktest
+
+// Now try to replay with libkleeRuntest but build the binary to use a different
+// bound for `klee_choose()`.
+// RUN: %cc -DBOUND_VALUE=2 -DPRINT_VALUE %s %libkleeruntest -Wl,-rpath=%libkleeruntestdir -o %t_runner
+
+// Check that the default is to exit with an error
+// RUN: not env KTEST_FILE=%t.klee-out/test000001.ktest %t_runner 2>&1 | FileCheck -check-prefix=CHECK_FATAL %s
+
+// Check that setting `KLEE_RUN_TEST_ERRORS_NON_FATAL` will not exit with an error
+// and will continue executing.
+// RUN: env KTEST_FILE=%t.klee-out/test000001.ktest KLEE_RUN_TEST_ERRORS_NON_FATAL=1 %t_runner 2>&1 | FileCheck %s
+
+#include "klee/klee.h"
+#include <stdint.h>
+#include <stdio.h>
+
+#ifndef BOUND_VALUE
+#error BOUND_VALUE must be defined
+#endif
+
+
+int main(int argc, char** argv) {
+  int x = klee_choose(BOUND_VALUE);
+#ifdef FORCE_VALUE
+  klee_assume(x == FORCE_VALUE);
+#endif
+
+#ifdef PRINT_VALUE
+  printf("x=%d\n", x);
+#endif
+
+  return 0;
+}
+// CHECK: KLEE_RUN_TEST_ERROR: klee_choose failure. max = 2, got = 20
+// CHECK: KLEE_RUN_TEST_ERROR: Forcing execution to continue
+// CHECK: x=20
+
+// CHECK_FATAL: KLEE_RUN_TEST_ERROR: klee_choose failure. max = 2, got = 20
+// CHECK_FATAL-NOT: x=20
+
diff --git a/test/Replay/libkleeruntest/replay_invalid_klee_range.c b/test/Replay/libkleeruntest/replay_invalid_klee_range.c
new file mode 100644
index 00000000..c7d62027
--- /dev/null
+++ b/test/Replay/libkleeruntest/replay_invalid_klee_range.c
@@ -0,0 +1,45 @@
+// RUN: %llvmgcc -DBOUND_VALUE=32 -DFORCE_VALUE=20 %s -emit-llvm -g -O0 -c -o %t.bc
+// RUN: rm -rf %t.klee-out
+// RUN: %klee --output-dir=%t.klee-out --libc=klee --search=dfs %t.bc
+// RUN: test -f %t.klee-out/test000001.ktest
+// RUN: test ! -f %t.klee-out/test000002.ktest
+
+// Now try to replay with libkleeRuntest but build the binary to use a different
+// bound for `klee_range()`.
+// RUN: %cc -DBOUND_VALUE=2 -DPRINT_VALUE %s %libkleeruntest -Wl,-rpath=%libkleeruntestdir -o %t_runner
+
+// Check that the default is to exit with an error
+// RUN: not env KTEST_FILE=%t.klee-out/test000001.ktest %t_runner 2>&1 | FileCheck -check-prefix=CHECK_FATAL %s
+
+// Check that setting `KLEE_RUN_TEST_ERRORS_NON_FATAL` will not exit with an error
+// and will continue executing.
+// RUN: env KTEST_FILE=%t.klee-out/test000001.ktest KLEE_RUN_TEST_ERRORS_NON_FATAL=1 %t_runner 2>&1 | FileCheck %s
+
+#include "klee/klee.h"
+#include <stdint.h>
+#include <stdio.h>
+
+#ifndef BOUND_VALUE
+#error BOUND_VALUE must be defined
+#endif
+
+
+int main(int argc, char** argv) {
+  int x = klee_range(0, BOUND_VALUE, "x");
+#ifdef FORCE_VALUE
+  klee_assume(x == FORCE_VALUE);
+#endif
+
+#ifdef PRINT_VALUE
+  printf("x=%d\n", x);
+#endif
+
+  return 0;
+}
+// CHECK: KLEE_RUN_TEST_ERROR: invalid klee_range(0,2,x) value, got: 20
+// CHECK: KLEE_RUN_TEST_ERROR: Forcing execution to continue
+// CHECK: x=20
+
+// CHECK_FATAL: KLEE_RUN_TEST_ERROR: invalid klee_range(0,2,x) value, got: 20
+// CHECK_FATAL-NOT: x=20
+
diff --git a/test/Replay/libkleeruntest/replay_invalid_num_objects.c b/test/Replay/libkleeruntest/replay_invalid_num_objects.c
new file mode 100644
index 00000000..43bc4867
--- /dev/null
+++ b/test/Replay/libkleeruntest/replay_invalid_num_objects.c
@@ -0,0 +1,39 @@
+// Compile program that only makes one klee_make_symbolic() call
+// RUN: %llvmgcc %s -emit-llvm -g -O0 -c -o %t.bc
+// RUN: rm -rf %t.klee-out
+// RUN: %klee --output-dir=%t.klee-out --search=dfs %t.bc
+// RUN: test -f %t.klee-out/test000001.ktest
+
+// Now try to replay with libkleeRuntest but build the binary so it
+// makes two calls to klee_make_symbolic.
+// RUN: %cc -DEXTRA_MAKE_SYMBOLIC %s %libkleeruntest -Wl,-rpath=%libkleeruntestdir -o %t_runner
+
+// Check that the default is to exit with an error
+// RUN: not env KTEST_FILE=%t.klee-out/test000001.ktest %t_runner 2>&1 | FileCheck -check-prefix=CHECK_FATAL %s
+
+// Check that setting `KLEE_RUN_TEST_ERRORS_NON_FATAL` will not exit with an error
+// and will continue executing.
+// RUN: env KTEST_FILE=%t.klee-out/test000001.ktest KLEE_RUN_TEST_ERRORS_NON_FATAL=1 %t_runner 2>&1 | FileCheck %s
+
+#include "klee/klee.h"
+#include <stdio.h>
+
+int main(int argc, char** argv) {
+  int x = 0;
+  klee_make_symbolic(&x, sizeof(x), "x");
+
+#ifdef EXTRA_MAKE_SYMBOLIC
+  int y = 1;
+  klee_make_symbolic(&y, sizeof(y), "x");
+  klee_assume(y == 0);
+  fprintf(stderr, "y is \"%d\"\n", y);
+#endif
+  return 0;
+}
+// CHECK: KLEE_RUN_TEST_ERROR: out of inputs
+// CHECK: KLEE_RUN_TEST_ERROR: Forcing execution to continue
+// CHECK: y is "0"
+
+// CHECK_FATAL: KLEE_RUN_TEST_ERROR: out of inputs
+// CHECK_FATAL-NOT: y is "0"
+
diff --git a/test/Replay/libkleeruntest/replay_invalid_object_names.c b/test/Replay/libkleeruntest/replay_invalid_object_names.c
new file mode 100644
index 00000000..9c75bebc
--- /dev/null
+++ b/test/Replay/libkleeruntest/replay_invalid_object_names.c
@@ -0,0 +1,45 @@
+// RUN: %llvmgcc -DOBJ_NAME=simple_name %s -emit-llvm -g -O0 -c -o %t.bc
+// RUN: rm -rf %t.klee-out
+// RUN: %klee --output-dir=%t.klee-out --search=dfs %t.bc
+// RUN: test -f %t.klee-out/test000001.ktest
+
+// Now try to replay with libkleeRuntest but build the binary to use a different
+// object name
+// RUN: %cc -DOBJ_NAME=wrong_name -DPRINT_VALUE %s %libkleeruntest -Wl,-rpath=%libkleeruntestdir -o %t_runner
+
+// Check that the default is to exit with an error
+// RUN: not env KTEST_FILE=%t.klee-out/test000001.ktest %t_runner 2>&1 | FileCheck -check-prefix=CHECK_FATAL %s
+
+// Check that setting `KLEE_RUN_TEST_ERRORS_NON_FATAL` will not exit with an error
+// and will continue executing.
+// RUN: env KTEST_FILE=%t.klee-out/test000001.ktest KLEE_RUN_TEST_ERRORS_NON_FATAL=1 %t_runner 2>&1 | FileCheck %s
+
+#include "klee/klee.h"
+#include <stdio.h>
+
+#ifndef OBJ_NAME
+#error OBJ_NAME must be defined
+#endif
+
+#define STRINGIFY(X) #X
+#define XSTRINGIFY(X) STRINGIFY(X)
+
+
+int main(int argc, char** argv) {
+  int x = 1;
+  klee_make_symbolic(&x, sizeof(x), XSTRINGIFY(OBJ_NAME));
+  klee_assume(x == 0);
+
+#ifdef PRINT_VALUE
+  printf("x=%d\n", x);
+#endif
+
+  return 0;
+}
+// CHECK: KLEE_RUN_TEST_ERROR: object name mismatch. Requesting "wrong_name" but returning "simple_name"
+// CHECK: KLEE_RUN_TEST_ERROR: Forcing execution to continue
+// CHECK: x=0
+
+// CHECK_FATAL: KLEE_RUN_TEST_ERROR: object name mismatch. Requesting "wrong_name" but returning "simple_name"
+// CHECK_FATAL-NOT: x=0
+
diff --git a/test/Replay/libkleeruntest/replay_invalid_object_size.c b/test/Replay/libkleeruntest/replay_invalid_object_size.c
new file mode 100644
index 00000000..a1513ef9
--- /dev/null
+++ b/test/Replay/libkleeruntest/replay_invalid_object_size.c
@@ -0,0 +1,43 @@
+// RUN: %llvmgcc -DINT_TYPE=uint8_t %s -emit-llvm -g -O0 -c -o %t.bc
+// RUN: rm -rf %t.klee-out
+// RUN: %klee --output-dir=%t.klee-out --search=dfs %t.bc
+// RUN: test -f %t.klee-out/test000001.ktest
+// RUN: test ! -f %t.klee-out/test000002.ktest
+
+// Now try to replay with libkleeRuntest but build the binary to use a different
+// size for variable `x`.
+// RUN: %cc -DINT_TYPE=uint32_t -DPRINT_VALUE %s %libkleeruntest -Wl,-rpath=%libkleeruntestdir -o %t_runner
+
+// Check that the default is to exit with an error
+// RUN: not env KTEST_FILE=%t.klee-out/test000001.ktest %t_runner 2>&1 | FileCheck -check-prefix=CHECK_FATAL %s
+
+// Check that setting `KLEE_RUN_TEST_ERRORS_NON_FATAL` will not exit with an error
+// and will continue executing.
+// RUN: env KTEST_FILE=%t.klee-out/test000001.ktest KLEE_RUN_TEST_ERRORS_NON_FATAL=1 %t_runner 2>&1 | FileCheck %s
+#include "klee/klee.h"
+#include <stdint.h>
+#include <stdio.h>
+
+#ifndef INT_TYPE
+#error INT_TYPE must be defined
+#endif
+
+
+int main(int argc, char** argv) {
+  INT_TYPE x = 1;
+  klee_make_symbolic(&x, sizeof(x), "x");
+  klee_assume(x == 0);
+
+#ifdef PRINT_VALUE
+  printf("x=%d\n", x);
+#endif
+
+  return 0;
+}
+// CHECK: KLEE_RUN_TEST_ERROR: object sizes differ. Expected 4 but got 1
+// CHECK: KLEE_RUN_TEST_ERROR: Forcing execution to continue
+// CHECK: x=0
+
+// CHECK_FATAL: KLEE_RUN_TEST_ERROR: object sizes differ. Expected 4 but got 1
+// CHECK_FATAL-NOT: x=0
+