diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/test.zig | 357 |
1 files changed, 357 insertions, 0 deletions
diff --git a/src/test.zig b/src/test.zig new file mode 100644 index 0000000..04d7f00 --- /dev/null +++ b/src/test.zig @@ -0,0 +1,357 @@ +//! Test runner for parameterized system tests +const ArrayList = std.ArrayList; +const ChildProcess = std.ChildProcess; +const LogLevel = std.log.Level; +const Progress = std.Progress; +const Server = std.zig.Server; +const allocator = std.testing.allocator; +const allocator_ref = &std.testing.allocator_instance; +const argsWithAllocator = std.process.argsWithAllocator; +const assert = std.debug.assert; +const builtin = @import("builtin"); +const concat = std.mem.concat; +const cwd = std.fs.cwd; +const dumpStackTrace = std.debug.dumpStackTrace; +const endsWith = std.mem.endsWith; +const eql = std.mem.eql; +const exit = std.process.exit; +const expectEqual = std.testing.expectEqual; +const expectEqualStrings = std.testing.expectEqualStrings; +const getStdIn = std.io.getStdIn; +const getStdOut = std.io.getStdOut; +const joinPath = std.fs.path.join; +const log_level = &std.testing.log_level; +const parseTarget = std.zig.CrossTarget.parse; +const print = std.debug.print; +const startsWith = std.mem.startsWith; +const std = @import("std"); +const target = builtin.target; +const test_io_mode = builtin.test_io_mode; +const tmpDir = std.testing.tmpDir; +const tokenizeScalar = std.mem.tokenizeScalar; +const trimRight = std.mem.trimRight; +const zig_version_string = builtin.zig_version_string; + +/// Options override. +pub const std_options = struct { + pub const io_mode = test_io_mode; + pub const logFn = struct { + pub fn log(comptime message_level: LogLevel, + comptime scope: @Type(.EnumLiteral), + comptime format: []const u8, args: anytype) void { + const fmt = "[" ++ @tagName(scope) ++ "] (" + ++ @tagName(message_level) ++ "): " ++ format ++ "\n"; + if (@intFromEnum(message_level) <= @intFromEnum(log_level.*)) + print(fmt, args); + } + }.log; +}; + +/// Print usage and exit with an error status. +fn printUsage() noreturn { + print("usage: zig build test -- [-r QBE] [FILE.ssa]... [DIR]...", .{}); + exit(1); +} + +/// QBE implementation to be tested. +var bin: []const u8 = undefined; +/// Reference implementation. +var bin_ref: ?[]const u8 = null; +/// Zig as a C compiler. +var zig: []const u8 = undefined; +/// Target name in Zig/Clang format. +var zig_target: []const u8 = undefined; +/// Target name in QBE format. +var qbe_target: []const u8 = undefined; +/// External executor for our implementation and output programs. +var emulator: ?[]const u8 = null; + +/// Parse command-line arguments. +fn parseArgs() ![]const []const u8 { + var listen = false; + defer if (!listen) + @panic("test not run in server mode"); + var paths = ArrayList([]const u8).init(allocator); + errdefer paths.deinit(); + + var args = try argsWithAllocator(allocator); + _ = args.next().?; // path to test binary + while (args.next()) |arg| { + if (eql(u8, arg, "--listen=-")) { + listen = true; + } else if (eql(u8, arg, "-b")) { + bin = try allocator.dupe(u8, args.next().?); + } else if (eql(u8, arg, "-r")) { + bin_ref = try allocator.dupe(u8, args.next().?); + } else if (eql(u8, arg, "-z")) { + zig = try allocator.dupe(u8, args.next().?); + } else if (eql(u8, arg, "-t")) { + const triple = args.next().?; + zig_target = try concat(allocator, u8, &.{ "--target=", triple }); + const cross_target = try parseTarget(.{ .arch_os_abi = triple }); + const os_tag = cross_target.os_tag orelse target.os.tag; + const cpu_arch = cross_target.cpu_arch orelse target.cpu.arch; + qbe_target = switch (os_tag) { + .macos => switch (cpu_arch) { + .aarch64 => "arm64_apple", + .x86_64 => "amd64_apple", + else => unreachable, + }, + else => switch (cpu_arch) { + .aarch64 => "arm64", + .riscv64 => "rv64", + .x86_64 => "amd64_sysv", + else => unreachable, + }, + }; + } else if (eql(u8, arg, "-e")) { + emulator = try allocator.dupe(u8, args.next().?); + } else if (endsWith(u8, arg, ".ssa")) { + const file = cwd().openFile(arg, .{}) catch |err| { + print("cannot open '{s}': {}\n", .{ arg, err }); + printUsage(); + }; + defer file.close(); + try paths.append(try allocator.dupe(u8, arg)); + } else { + var parent = cwd().openIterableDir(arg, .{}) catch |err| { + print("cannot open '{s}': {}\n", .{ arg, err }); + printUsage(); + }; + defer parent.close(); + var it = parent.iterate(); + while (try it.next()) |file| { + if (file.kind != .file) continue; + if (startsWith(u8, file.name, "_")) continue; + if (!endsWith(u8, file.name, ".ssa")) continue; + try paths.append(try joinPath(allocator, &.{ arg, file.name })); + } + } + } + + if (paths.items.len == 0) { + print("no test to run.\n", .{}); + printUsage(); + } + return paths.toOwnedSlice(); +} + +/// Allocate and initialize every item to 0. +fn calloc(comptime T: type, n: usize) ![]T { + const slice = try allocator.alloc(T, n); + for (slice) |*item| + item.* = 0; + return slice; +} + +/// Extract embedded test driver and output. +fn extract(ssa: []const u8, drv: []const u8, out: []const u8) !struct { + driver: bool, + output: bool, +} { + const dir = cwd(); + const source = try dir.openFile(ssa, .{}); + defer source.close(); + const driver = try dir.createFile(drv, .{}); + defer driver.close(); + const output = try dir.createFile(out, .{}); + defer output.close(); + var has_driver = false; + var has_output = false; + + const ir = try source.readToEndAlloc(allocator, 1 << 20); + defer allocator.free(ir); + var lines = tokenizeScalar(u8, ir, '\n'); + while (lines.next()) |line| + if (startsWith(u8, line, "# skip ")) { + var skipped = tokenizeScalar(u8, line["# skip ".len..], ' '); + while (skipped.next()) |s| + if (eql(u8, s, qbe_target)) + return error.SkipZigTest; + } else if (startsWith(u8, line, "# >>> ")) { + const writer = if (eql(u8, line, "# >>> driver")) w: { + has_driver = true; + break :w driver.writer(); + } else if (eql(u8, line, "# >>> output")) w: { + has_output = true; + break :w output.writer(); + } else continue; + while (lines.next()) |comment| { + if (eql(u8, comment, "# <<<")) + break; + if (!startsWith(u8, comment, "# ")) + return error.InvalidDocTest; + try writer.print("{s}\n", .{ + trimRight(u8, comment["# ".len..], "#"), + }); + } + }; + return .{ .driver = has_driver, .output = has_output }; +} + +/// Spawn given command and return the collected stdout. +fn collectOutput(argv: []const []const u8) ![]const u8 { + const result = try ChildProcess.exec(.{ + .allocator = allocator, + .argv = argv, + .max_output_bytes = 1 << 20, + }); + allocator.free(result.stderr); + return result.stdout; +} + +/// Spawn given command and expect a non-error exit status. +fn spawn(argv: []const []const u8) !void { + var proc = ChildProcess.init(argv, allocator); + proc.stderr_behavior = .Ignore; + try expectEqual(ChildProcess.Term{ .Exited = 0 }, try proc.spawnAndWait()); +} + +/// Count the number of lines in a buffer and free it. +fn loc(buffer: []const u8) u32 { + var n: u32 = 0; + for (buffer) |c| { + if (c == '\n') + n += 1; + } else allocator.free(buffer); + return n; +} + +/// Compile given QBE IR and check its exit status and (optionally) stdout. +fn check(ssa: []const u8) anyerror!void { + var tmp = tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmp.dir.realpathAlloc(allocator, "."); + defer allocator.free(dir); + const drv = try joinPath(allocator, &.{ dir, "driver.c" }); + defer allocator.free(drv); + const out = try joinPath(allocator, &.{ dir, "output" }); + defer allocator.free(out); + const extracted = try extract(ssa, drv, out); + + const s = try joinPath(allocator, &.{ dir, "a.s" }); + defer allocator.free(s); + const ir_loc = b: { + const ir = try collectOutput(if (emulator) |executor| + &.{ executor, bin, ssa } + else + &.{ bin, ssa }); + errdefer allocator.free(ir); + try tmp.parent_dir.writeFile(s, ir); + break :b loc(ir); + }; + if (bin_ref) |qbe| { + const loc_diff = @as(i64, ir_loc) - loc(try collectOutput(&.{ + qbe, "-t", qbe_target, ssa + })); + if (loc_diff != 0) + print("'{s}': {:1} lines asm\n", .{ ssa, loc_diff }); + } + + const exe = try joinPath(allocator, &.{ dir, "a.out" }); + defer allocator.free(exe); + try spawn(if (extracted.driver) + &.{ zig, "cc", zig_target, "-g", "-o", exe, drv, s } + else + &.{ zig, "cc", zig_target, "-g", "-o", exe, s }); + + const run_argv: []const []const u8 = if (emulator) |executor| + &.{ executor, exe, "a", "b", "c" } + else + &.{ exe, "a", "b", "c" }; + if (extracted.output) { + const expected = try tmp.dir.readFileAlloc(allocator, out, 1 << 20); + defer allocator.free(expected); + const actual = try collectOutput(run_argv); + defer allocator.free(actual); + try expectEqualStrings(expected, actual); + } else try spawn(run_argv); +} + +/// Collect paths to test QBE IR files and feed them to the fn check +/// instead of running builtin.test_functions. +pub fn main() !void { + const paths = try parseArgs(); + defer { + for (paths) |ssa| allocator.free(ssa); + allocator.free(paths); + // Global variables + allocator.free(bin); + if (bin_ref) |string| allocator.free(string); + allocator.free(zig); + allocator.free(zig_target); + if (emulator) |string| allocator.free(string); + assert(allocator_ref.deinit() == .ok); + } + var server = try Server.init(.{ + .gpa = allocator, + .in = getStdIn(), + .out = getStdOut(), + .zig_version = zig_version_string, + }); + defer server.deinit(); + + while (server.receiveMessage()) |message| { + switch (message.tag) { + .query_test_metadata => { + const parent_allocator = allocator_ref.*; + allocator_ref.* = .{}; + defer allocator_ref.* = parent_allocator; + defer assert(allocator_ref.deinit() == .ok); + + var string_bytes = ArrayList(u8).init(allocator); + defer string_bytes.deinit(); + const names = try allocator.alloc(u32, paths.len); + defer allocator.free(names); + for (paths, names) |ssa, *name| { + name.* = @intCast(string_bytes.items.len); + try string_bytes.ensureUnusedCapacity(ssa.len + 1); + string_bytes.appendSliceAssumeCapacity(ssa); + string_bytes.appendAssumeCapacity(0); + } + + const async_frame_sizes = try calloc(u32, paths.len); + defer allocator.free(async_frame_sizes); + const expected_panic_msgs = try calloc(u32, paths.len); + defer allocator.free(expected_panic_msgs); + + try server.serveTestMetadata(.{ + .names = names, + .async_frame_sizes = async_frame_sizes, + .expected_panic_msgs = expected_panic_msgs, + .string_bytes = string_bytes.items, + }); + }, + .run_test => { + const parent_allocator = allocator_ref.*; + allocator_ref.* = .{}; + defer allocator_ref.* = parent_allocator; + + const index = try server.receiveBody_u32(); + var fail = false; + var skip = false; + check(paths[index]) catch |err| switch (err) { + error.SkipZigTest => skip = true, + else => { + fail = true; + if (@errorReturnTrace()) |trace| + dumpStackTrace(trace.*); + }, + }; + try server.serveTestResults(.{ + .index = index, + .flags = .{ + .fail = fail, + .skip = skip, + .leak = allocator_ref.deinit() == .leak, + }, + }); + }, + .exit => break, + else => |tag| { + print("unsupported message: {}", .{ tag }); + exit(1); + }, + } + } else |err| return err; +} |