Port test runner to Zig

5 files changed, 376 insertions, 271 deletions

diff --git a/.gitignore b/.gitignore
index 6d548b5..e73c965 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,2 @@
 zig-cache/
 zig-out/
-qbe
diff --git a/Makefile b/Makefile
deleted file mode 100644
index a81e46d..0000000
--- a/Makefile
+++ /dev/null
@@ -1,42 +0,0 @@
-.POSIX:
-.SUFFIXES: .o .c
-
-PREFIX = /usr/local
-BINDIR = $(PREFIX)/bin
-
-ZIGSRC   = src/main.zig src/cimport.zig
-COMMSRC  = util.c parse.c abi.c cfg.c mem.c ssa.c alias.c load.c \
-           copy.c fold.c simpl.c live.c spill.c rega.c emit.c
-AMD64SRC = amd64/targ.c amd64/sysv.c amd64/isel.c amd64/emit.c
-ARM64SRC = arm64/targ.c arm64/abi.c arm64/isel.c arm64/emit.c
-RV64SRC  = rv64/targ.c rv64/abi.c rv64/isel.c rv64/emit.c
-SRCALL   = $(ZIGSRC) $(COMMSRC) $(AMD64SRC) $(ARM64SRC) $(RV64SRC)
-
-zig-out/bin/roux: $(SRCALL) all.h ops.h amd64/all.h arm64/all.h rv64/all.h
-	zig build
-
-install: roux
-	zig build install --prefix="$(DESTDIR)$(BINDIR)"
-
-uninstall:
-	zig build uninstall --prefix="$(DESTDIR)$(BINDIR)"
-
-clean:
-	rm -fr qbe zig-cache zig-out
-
-qbe: zig-out/bin/roux
-	ln -fs $< $@
-
-check: qbe
-	tools/test.sh all
-
-check-arm64: qbe
-	TARGET=arm64 tools/test.sh all
-
-check-rv64: qbe
-	TARGET=rv64 tools/test.sh all
-
-src:
-	@echo $(SRCALL)
-
-.PHONY: clean check check-arm64 check-rv64 src install uninstall
diff --git a/build.zig b/build.zig
index 7609463..0ac29d5 100644
--- a/build.zig
+++ b/build.zig
@@ -1,7 +1,9 @@
 //! Build recipe
+const Build = std.Build;
+const getExternalExecutor = std.zig.system.NativeTargetInfo.getExternalExecutor;
 const std = @import("std");
 
-pub fn build(b: *std.Build) void {
+pub fn build(b: *Build) !void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
 
@@ -35,13 +37,22 @@ pub fn build(b: *std.Build) void {
         run_cmd.addArgs(args);
     b.step("run", "Run the app").dependOn(&run_cmd.step);
 
-    const unit_tests = b.addTest(.{
-        .root_source_file = .{ .path = "src/main.zig" },
-        .target = target,
+    const tests = b.addTest(.{
+        .root_source_file = .{ .path = "src/test.zig" },
         .optimize = optimize,
+        .test_runner = "src/test.zig",
     });
-
-    const run_unit_tests = b.addRunArtifact(unit_tests);
-    const test_step = b.step("test", "Run unit tests");
-    test_step.dependOn(&run_unit_tests.step);
+    const run_tests = b.addRunArtifact(tests);
+    run_tests.addArg("-b");
+    run_tests.addArtifactArg(bin);
+    run_tests.addArgs(&.{ "-z", b.zig_exe });
+    run_tests.addArgs(&.{ "-t", try target.zigTriple(b.allocator) });
+    switch (getExternalExecutor(tests.target_info, bin.target_info, .{})) {
+        .qemu => |emulator| run_tests.addArgs(&.{ "-e", emulator }),
+        else => {},
+    }
+    if (b.args) |args|
+        run_tests.addArgs(args);
+    run_tests.has_side_effects = true;
+    b.step("test", "Run unit tests").dependOn(&run_tests.step);
 }
diff --git a/src/test.zig b/src/test.zig
new file mode 100644
index 0000000..04d7f00
--- /dev/null
+++ b/src/test.zig
@@ -0,0 +1,357 @@
+//! Test runner for parameterized system tests
+const ArrayList = std.ArrayList;
+const ChildProcess = std.ChildProcess;
+const LogLevel = std.log.Level;
+const Progress = std.Progress;
+const Server = std.zig.Server;
+const allocator = std.testing.allocator;
+const allocator_ref = &std.testing.allocator_instance;
+const argsWithAllocator = std.process.argsWithAllocator;
+const assert = std.debug.assert;
+const builtin = @import("builtin");
+const concat = std.mem.concat;
+const cwd = std.fs.cwd;
+const dumpStackTrace = std.debug.dumpStackTrace;
+const endsWith = std.mem.endsWith;
+const eql = std.mem.eql;
+const exit = std.process.exit;
+const expectEqual = std.testing.expectEqual;
+const expectEqualStrings = std.testing.expectEqualStrings;
+const getStdIn = std.io.getStdIn;
+const getStdOut = std.io.getStdOut;
+const joinPath = std.fs.path.join;
+const log_level = &std.testing.log_level;
+const parseTarget = std.zig.CrossTarget.parse;
+const print = std.debug.print;
+const startsWith = std.mem.startsWith;
+const std = @import("std");
+const target = builtin.target;
+const test_io_mode = builtin.test_io_mode;
+const tmpDir = std.testing.tmpDir;
+const tokenizeScalar = std.mem.tokenizeScalar;
+const trimRight = std.mem.trimRight;
+const zig_version_string = builtin.zig_version_string;
+
+/// Options override.
+pub const std_options = struct {
+    pub const io_mode = test_io_mode;
+    pub const logFn = struct {
+        pub fn log(comptime message_level: LogLevel,
+                   comptime scope: @Type(.EnumLiteral),
+                   comptime format: []const u8, args: anytype) void {
+            const fmt = "[" ++ @tagName(scope) ++ "] ("
+                ++ @tagName(message_level) ++ "): " ++ format ++ "\n";
+            if (@intFromEnum(message_level) <= @intFromEnum(log_level.*))
+                print(fmt, args);
+        }
+    }.log;
+};
+
+/// Print usage and exit with an error status.
+fn printUsage() noreturn {
+    print("usage: zig build test -- [-r QBE] [FILE.ssa]... [DIR]...", .{});
+    exit(1);
+}
+
+/// QBE implementation to be tested.
+var bin: []const u8 = undefined;
+/// Reference implementation.
+var bin_ref: ?[]const u8 = null;
+/// Zig as a C compiler.
+var zig: []const u8 = undefined;
+/// Target name in Zig/Clang format.
+var zig_target: []const u8 = undefined;
+/// Target name in QBE format.
+var qbe_target: []const u8 = undefined;
+/// External executor for our implementation and output programs.
+var emulator: ?[]const u8 = null;
+
+/// Parse command-line arguments.
+fn parseArgs() ![]const []const u8 {
+    var listen = false;
+    defer if (!listen)
+        @panic("test not run in server mode");
+    var paths = ArrayList([]const u8).init(allocator);
+    errdefer paths.deinit();
+
+    var args = try argsWithAllocator(allocator);
+    _ = args.next().?; // path to test binary
+    while (args.next()) |arg| {
+        if (eql(u8, arg, "--listen=-")) {
+            listen = true;
+        } else if (eql(u8, arg, "-b")) {
+            bin = try allocator.dupe(u8, args.next().?);
+        } else if (eql(u8, arg, "-r")) {
+            bin_ref = try allocator.dupe(u8, args.next().?);
+        } else if (eql(u8, arg, "-z")) {
+            zig = try allocator.dupe(u8, args.next().?);
+        } else if (eql(u8, arg, "-t")) {
+            const triple = args.next().?;
+            zig_target = try concat(allocator, u8, &.{ "--target=", triple });
+            const cross_target = try parseTarget(.{ .arch_os_abi = triple });
+            const os_tag = cross_target.os_tag orelse target.os.tag;
+            const cpu_arch = cross_target.cpu_arch orelse target.cpu.arch;
+            qbe_target = switch (os_tag) {
+                .macos => switch (cpu_arch) {
+                    .aarch64 => "arm64_apple",
+                    .x86_64 => "amd64_apple",
+                    else => unreachable,
+                },
+                else => switch (cpu_arch) {
+                    .aarch64 => "arm64",
+                    .riscv64 => "rv64",
+                    .x86_64 => "amd64_sysv",
+                    else => unreachable,
+                },
+            };
+        } else if (eql(u8, arg, "-e")) {
+            emulator = try allocator.dupe(u8, args.next().?);
+        } else if (endsWith(u8, arg, ".ssa")) {
+            const file = cwd().openFile(arg, .{}) catch |err| {
+                print("cannot open '{s}': {}\n", .{ arg, err });
+                printUsage();
+            };
+            defer file.close();
+            try paths.append(try allocator.dupe(u8, arg));
+        } else {
+            var parent = cwd().openIterableDir(arg, .{}) catch |err| {
+                print("cannot open '{s}': {}\n", .{ arg, err });
+                printUsage();
+            };
+            defer parent.close();
+            var it = parent.iterate();
+            while (try it.next()) |file| {
+                if (file.kind != .file) continue;
+                if (startsWith(u8, file.name, "_")) continue;
+                if (!endsWith(u8, file.name, ".ssa")) continue;
+                try paths.append(try joinPath(allocator, &.{ arg, file.name }));
+            }
+        }
+    }
+
+    if (paths.items.len == 0) {
+        print("no test to run.\n", .{});
+        printUsage();
+    }
+    return paths.toOwnedSlice();
+}
+
+/// Allocate and initialize every item to 0.
+fn calloc(comptime T: type, n: usize) ![]T {
+    const slice = try allocator.alloc(T, n);
+    for (slice) |*item|
+        item.* = 0;
+    return slice;
+}
+
+/// Extract embedded test driver and output.
+fn extract(ssa: []const u8, drv: []const u8, out: []const u8) !struct {
+    driver: bool,
+    output: bool,
+} {
+    const dir = cwd();
+    const source = try dir.openFile(ssa, .{});
+    defer source.close();
+    const driver = try dir.createFile(drv, .{});
+    defer driver.close();
+    const output = try dir.createFile(out, .{});
+    defer output.close();
+    var has_driver = false;
+    var has_output = false;
+
+    const ir = try source.readToEndAlloc(allocator, 1 << 20);
+    defer allocator.free(ir);
+    var lines = tokenizeScalar(u8, ir, '\n');
+    while (lines.next()) |line|
+        if (startsWith(u8, line, "# skip ")) {
+            var skipped = tokenizeScalar(u8, line["# skip ".len..], ' ');
+            while (skipped.next()) |s|
+                if (eql(u8, s, qbe_target))
+                    return error.SkipZigTest;
+        } else if (startsWith(u8, line, "# >>> ")) {
+            const writer = if (eql(u8, line, "# >>> driver")) w: {
+                has_driver = true;
+                break :w driver.writer();
+            } else if (eql(u8, line, "# >>> output")) w: {
+                has_output = true;
+                break :w output.writer();
+            } else continue;
+            while (lines.next()) |comment| {
+                if (eql(u8, comment, "# <<<"))
+                    break;
+                if (!startsWith(u8, comment, "# "))
+                    return error.InvalidDocTest;
+                try writer.print("{s}\n", .{
+                    trimRight(u8, comment["# ".len..], "#"),
+                });
+            }
+        };
+    return .{ .driver = has_driver, .output = has_output };
+}
+
+/// Spawn given command and return the collected stdout.
+fn collectOutput(argv: []const []const u8) ![]const u8 {
+    const result = try ChildProcess.exec(.{
+        .allocator = allocator,
+        .argv = argv,
+        .max_output_bytes = 1 << 20,
+    });
+    allocator.free(result.stderr);
+    return result.stdout;
+}
+
+/// Spawn given command and expect a non-error exit status.
+fn spawn(argv: []const []const u8) !void {
+    var proc = ChildProcess.init(argv, allocator);
+    proc.stderr_behavior = .Ignore;
+    try expectEqual(ChildProcess.Term{ .Exited = 0 }, try proc.spawnAndWait());
+}
+
+/// Count the number of lines in a buffer and free it.
+fn loc(buffer: []const u8) u32 {
+    var n: u32 = 0;
+    for (buffer) |c| {
+        if (c == '\n')
+            n += 1;
+    } else allocator.free(buffer);
+    return n;
+}
+
+/// Compile given QBE IR and check its exit status and (optionally) stdout.
+fn check(ssa: []const u8) anyerror!void {
+    var tmp = tmpDir(.{});
+    defer tmp.cleanup();
+    const dir = try tmp.dir.realpathAlloc(allocator, ".");
+    defer allocator.free(dir);
+    const drv = try joinPath(allocator, &.{ dir, "driver.c" });
+    defer allocator.free(drv);
+    const out = try joinPath(allocator, &.{ dir, "output" });
+    defer allocator.free(out);
+    const extracted = try extract(ssa, drv, out);
+
+    const s = try joinPath(allocator, &.{ dir, "a.s" });
+    defer allocator.free(s);
+    const ir_loc = b: {
+        const ir = try collectOutput(if (emulator) |executor|
+            &.{ executor, bin, ssa }
+        else
+            &.{ bin, ssa });
+        errdefer allocator.free(ir);
+        try tmp.parent_dir.writeFile(s, ir);
+        break :b loc(ir);
+    };
+    if (bin_ref) |qbe| {
+        const loc_diff = @as(i64, ir_loc) - loc(try collectOutput(&.{
+            qbe, "-t", qbe_target, ssa
+        }));
+        if (loc_diff != 0)
+            print("'{s}': {:1} lines asm\n", .{ ssa, loc_diff });
+    }
+
+    const exe = try joinPath(allocator, &.{ dir, "a.out" });
+    defer allocator.free(exe);
+    try spawn(if (extracted.driver)
+        &.{ zig, "cc", zig_target, "-g", "-o", exe, drv, s }
+    else
+        &.{ zig, "cc", zig_target, "-g", "-o", exe, s });
+
+    const run_argv: []const []const u8 = if (emulator) |executor|
+        &.{ executor, exe, "a", "b", "c" }
+    else
+        &.{ exe, "a", "b", "c" };
+    if (extracted.output) {
+        const expected = try tmp.dir.readFileAlloc(allocator, out, 1 << 20);
+        defer allocator.free(expected);
+        const actual = try collectOutput(run_argv);
+        defer allocator.free(actual);
+        try expectEqualStrings(expected, actual);
+    } else try spawn(run_argv);
+}
+
+/// Collect paths to test QBE IR files and feed them to the fn check
+/// instead of running builtin.test_functions.
+pub fn main() !void {
+    const paths = try parseArgs();
+    defer {
+        for (paths) |ssa| allocator.free(ssa);
+        allocator.free(paths);
+        // Global variables
+        allocator.free(bin);
+        if (bin_ref) |string| allocator.free(string);
+        allocator.free(zig);
+        allocator.free(zig_target);
+        if (emulator) |string| allocator.free(string);
+        assert(allocator_ref.deinit() == .ok);
+    }
+    var server = try Server.init(.{
+        .gpa = allocator,
+        .in = getStdIn(),
+        .out = getStdOut(),
+        .zig_version = zig_version_string,
+    });
+    defer server.deinit();
+
+    while (server.receiveMessage()) |message| {
+        switch (message.tag) {
+            .query_test_metadata => {
+                const parent_allocator = allocator_ref.*;
+                allocator_ref.* = .{};
+                defer allocator_ref.* = parent_allocator;
+                defer assert(allocator_ref.deinit() == .ok);
+
+                var string_bytes = ArrayList(u8).init(allocator);
+                defer string_bytes.deinit();
+                const names = try allocator.alloc(u32, paths.len);
+                defer allocator.free(names);
+                for (paths, names) |ssa, *name| {
+                    name.* = @intCast(string_bytes.items.len);
+                    try string_bytes.ensureUnusedCapacity(ssa.len + 1);
+                    string_bytes.appendSliceAssumeCapacity(ssa);
+                    string_bytes.appendAssumeCapacity(0);
+                }
+
+                const async_frame_sizes = try calloc(u32, paths.len);
+                defer allocator.free(async_frame_sizes);
+                const expected_panic_msgs = try calloc(u32, paths.len);
+                defer allocator.free(expected_panic_msgs);
+
+                try server.serveTestMetadata(.{
+                    .names = names,
+                    .async_frame_sizes = async_frame_sizes,
+                    .expected_panic_msgs = expected_panic_msgs,
+                    .string_bytes = string_bytes.items,
+                });
+            },
+            .run_test => {
+                const parent_allocator = allocator_ref.*;
+                allocator_ref.* = .{};
+                defer allocator_ref.* = parent_allocator;
+
+                const index = try server.receiveBody_u32();
+                var fail = false;
+                var skip = false;
+                check(paths[index]) catch |err| switch (err) {
+                    error.SkipZigTest => skip = true,
+                    else => {
+                        fail = true;
+                        if (@errorReturnTrace()) |trace|
+                            dumpStackTrace(trace.*);
+                    },
+                };
+                try server.serveTestResults(.{
+                    .index = index,
+                    .flags = .{
+                        .fail = fail,
+                        .skip = skip,
+                        .leak = allocator_ref.deinit() == .leak,
+                    },
+                });
+            },
+            .exit => break,
+            else => |tag| {
+                print("unsupported message: {}", .{ tag });
+                exit(1);
+            },
+        }
+    } else |err| return err;
+}
diff --git a/tools/test.sh b/tools/test.sh
deleted file mode 100755
index 23c6663..0000000
--- a/tools/test.sh
+++ /dev/null
@@ -1,220 +0,0 @@
-#!/bin/sh
-
-dir=`dirname "$0"`
-bin=$dir/../qbe
-binref=$dir/../qbe.ref
-
-tmp=/tmp/qbe.zzzz
-
-drv=$tmp.c
-asm=$tmp.s
-asmref=$tmp.ref.s
-exe=$tmp.exe
-out=$tmp.out
-
-init() {
-	case "$TARGET" in
-	arm64)
-		for p in aarch64-linux-musl aarch64-linux-gnu
-		do
-			cc="$p-gcc -no-pie -static"
-			qemu="qemu-aarch64"
-			if
-				$cc -v >/dev/null 2>&1 &&
-				$qemu -version >/dev/null 2>&1
-			then
-				if sysroot=$($cc -print-sysroot) && test -n "$sysroot"
-				then
-					qemu="$qemu -L $sysroot"
-				fi
-				break
-			fi
-			cc=
-		done
-		if test -z "$cc"
-		then
-			echo "Cannot find arm64 compiler or qemu."
-			exit 1
-		fi
-		bin="$bin -t arm64"
-		;;
-	rv64)
-		for p in riscv64-linux-musl riscv64-linux-gnu
-		do
-			cc="$p-gcc -no-pie -static"
-			qemu="qemu-riscv64"
-			if
-				$cc -v >/dev/null 2>&1 &&
-				$qemu -version >/dev/null 2>&1
-			then
-				if sysroot=$($cc -print-sysroot) && test -n "$sysroot"
-				then
-					qemu="$qemu -L $sysroot"
-				fi
-				break
-			fi
-			cc=
-		done
-		if test -z "$cc"
-		then
-			echo "Cannot find riscv64 compiler or qemu."
-			exit 1
-		fi
-		bin="$bin -t rv64"
-		;;
-	"")
-		case `uname` in
-		*Darwin*)
-			cc="cc"
-			;;
-		*OpenBSD*)
-			cc="cc -nopie -lpthread"
-			;;
-		*FreeBSD*)
-			cc="cc -lpthread"
-			;;
-		*)
-			cc="${CC:-cc}"
-			ccpost="-lpthread"
-			;;
-		esac
-		TARGET=`$bin -t?`
-		;;
-	*)
-		echo "Unknown target '$TARGET'."
-		exit 1
-		;;
-	esac
-}
-
-cleanup() {
-	rm -f $drv $asm $exe $out
-}
-
-extract() {
-	WHAT="$1"
-	FILE="$2"
-
-	awk "
-		/^# >>> $WHAT/ {
-			p = 1
-			next
-		}
-		/^# <<</ {
-			p = 0
-		}
-		p
-	" $FILE \
-	| sed -e 's/# //' \
-	| sed -e 's/#$//'
-}
-
-once() {
-	t="$1"
-
-	if ! test -f $t
-	then
-		echo "invalid test file $t" >&2
-		exit 1
-	fi
-
-	if
-		sed -e 1q $t |
-		grep "skip.* $TARGET\( .*\)\?$" \
-		>/dev/null
-	then
-		return 0
-	fi
-
-	printf "%-45s" "$(basename $t)..."
-
-	if ! $bin -o $asm $t
-	then
-		echo "[qbe fail]"
-		return 1
-	fi
-
-	if test -x $binref
-	then
-		$binref -o $asmref $t 2>/dev/null
-	fi
-
-	extract driver $t > $drv
-	extract output $t > $out
-
-	if test -s $drv
-	then
-		src="$drv $asm"
-	else
-		src="$asm"
-	fi
-
-	if ! $cc -g -o $exe $src $ccpost
-	then
-		echo "[cc fail]"
-		return 1
-	fi
-
-	if test -s $out
-	then
-		$qemu $exe a b c | diff -u - $out
-		ret=$?
-		reason="output"
-	else
-		$qemu $exe a b c
-		ret=$?
-		reason="returned $ret"
-	fi
-
-	if test $ret -ne 0
-	then
-		echo "[$reason fail]"
-		return 1
-	fi
-
-	echo "[ok]"
-
-	if test -f $asmref && ! cmp -s $asm $asmref
-	then
-		loc0=`wc -l $asm    | cut -d' ' -f1`
-		loc1=`wc -l $asmref | cut -d' ' -f1`
-		printf "    asm diff: %+d\n" $(($loc0 - $loc1))
-		return 0
-	fi
-}
-
-#trap cleanup TERM QUIT
-
-init
-
-if test -z "$1"
-then
-	echo "usage: tools/test.sh {all, SSAFILE}" 2>&1
-	exit 1
-fi
-
-case "$1" in
-"all")
-	fail=0
-	count=0
-	for t in $dir/../test/[!_]*.ssa
-	do
-		once $t
-		fail=`expr $fail + $?`
-		count=`expr $count + 1`
-	done
-	if test $fail -ge 1
-	then
-		echo
-		echo "$fail of $count tests failed!"
-	else
-		echo
-		echo "All is fine!"
-	fi
-	exit $fail
-	;;
-*)
-	once $1
-	exit $?
-	;;
-esac