From 16993bba8fa359b093d44fe395044bfd392c19f3 Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sat, 11 Nov 2023 01:45:13 +0000 Subject: benchmark: Add support for COMPARISON file --- benchmark/COMPARISON | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index 55ab94b4..d8750e34 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,4 +1,4 @@ -CPU | Mz | exec/s | afl-*-config | -========================================|======|========|==============| -CPU 12th Gen Intel(R) Core(TM) i7-1270P | 4200 | 12750 | both | -AMD EPYC 7282 16-Core Processor | 3190 | 10060 | both | +CPU | MHz | singlecore | multicore | afl-*-config | +===============================|=======|============|===========|==============| +Apple Mac Studio M2 Ultra 2023 | 3500 | 174386 | 1112585 | both | +Intel(R) Core(TM) i9-9900K CPU | 4999 | 133706 | 1169989 | both | -- cgit 1.4.1 From 8b79d9b4d5efdf4d0e30a323dd3a169f9a7801c5 Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sun, 12 Nov 2023 07:40:58 -0800 Subject: benchmark: show the number of cores used in COMPARISON --- benchmark/COMPARISON | 8 ++++---- benchmark/benchmark.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index d8750e34..03a197cd 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,4 +1,4 @@ -CPU | MHz | singlecore | multicore | afl-*-config | -===============================|=======|============|===========|==============| -Apple Mac Studio M2 Ultra 2023 | 3500 | 174386 | 1112585 | both | -Intel(R) Core(TM) i9-9900K CPU | 4999 | 133706 | 1169989 | both | +CPU | MHz | threads | singlecore | multicore | afl-*-config | +===============================|=======|=========|============|===========|==============| +Apple Mac Studio M2 Ultra 2023 | 3500 | 16 | 174386 | 1112585 | both | +Intel(R) Core(TM) i9-9900K CPU | 4995 | 16 | 133706 | 1169989 | both | diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 6fde621b..2601ef0d 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -195,6 +195,7 @@ async def save_benchmark_results() -> None: print(blue(f" [*] Results have been written to {jsonfile.name}")) with open("COMPARISON", "a") as comparisonfile: described_config = await describe_afl_config() + aflconfig = described_config.ljust(12) if results.hardware is None: return cpu_model = results.hardware.cpu_model.ljust(42) @@ -205,9 +206,10 @@ async def save_benchmark_results() -> None: return single = str(round(results.targets["test-instr-persist-shmem"]["singlecore"].total_execs_per_sec)).ljust(10) multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].total_execs_per_sec)).ljust(9) + cores = str(args.fuzzers).ljust(7) if len(cpu_model) > 30: cpu_model = cpu_model[:30] - comparisonfile.write(f"{cpu_model} | {cpu_mhz} | {single} | {multi} | {described_config.ljust(12)} |\n") + comparisonfile.write(f"{cpu_model} | {cpu_mhz} | {cores} | {single} | {multi} | {aflconfig} |\n") with open("COMPARISON", "r") as comparisonfile: print(comparisonfile.read()) -- cgit 1.4.1 From 26045831a23ceef70834c7a7be4d17d436a4cf8e Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sun, 12 Nov 2023 11:52:17 -0800 Subject: benchmark: use afl's execs/s; increase CPU model width --- benchmark/COMPARISON | 8 ++++---- benchmark/benchmark.py | 9 +++------ 2 files changed, 7 insertions(+), 10 deletions(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index 03a197cd..c26c6adb 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,4 +1,4 @@ -CPU | MHz | threads | singlecore | multicore | afl-*-config | -===============================|=======|=========|============|===========|==============| -Apple Mac Studio M2 Ultra 2023 | 3500 | 16 | 174386 | 1112585 | both | -Intel(R) Core(TM) i9-9900K CPU | 4995 | 16 | 133706 | 1169989 | both | +CPU | MHz | threads | singlecore | multicore | afl-*-config | +====================================================|=======|=========|============|===========|==============| +Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 174386 | 1112585 | both | +Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 133706 | 1169989 | both | diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 5b363e16..1510acb6 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -198,17 +198,15 @@ async def save_benchmark_results() -> None: aflconfig = described_config.ljust(12) if results.hardware is None: return - cpu_model = results.hardware.cpu_model.ljust(42) + cpu_model = results.hardware.cpu_model.ljust(51) cpu_mhz = str(round(results.hardware.cpu_fastest_core_mhz)).ljust(5) if "test-instr-persist-shmem" in results.targets and "multicore" in results.targets["test-instr-persist-shmem"]: if results.targets["test-instr-persist-shmem"]["singlecore"] is None or \ results.targets["test-instr-persist-shmem"]["multicore"] is None: return - single = str(round(results.targets["test-instr-persist-shmem"]["singlecore"].total_execs_per_sec)).ljust(10) - multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].total_execs_per_sec)).ljust(9) + single = str(round(results.targets["test-instr-persist-shmem"]["singlecore"].afl_execs_per_sec)).ljust(10) + multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].afl_execs_per_sec)).ljust(9) cores = str(args.fuzzers).ljust(7) - if len(cpu_model) > 30: - cpu_model = cpu_model[:30] comparisonfile.write(f"{cpu_model} | {cpu_mhz} | {cores} | {single} | {multi} | {aflconfig} |\n") with open("COMPARISON", "r") as comparisonfile: print(comparisonfile.read()) @@ -281,7 +279,6 @@ async def main() -> None: results.targets[binary][mode.name] = run print(f" [*] Average AFL execs/sec for this test across all runs was: {green(avg_afl_execs_per_sec)}") - print(f" [*] Average total execs/sec for this test across all runs was: {green(total_execs_per_sec)}") if (((max(afl_execs_per_sec) - min(afl_execs_per_sec)) / avg_afl_execs_per_sec) * 100) > 15: print(yellow(" [*] The difference between your slowest and fastest runs was >15%, maybe try again?")) -- cgit 1.4.1 From 75a3af8a23f2fd1488ee7cb6e5fca00289031c6c Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sun, 19 Nov 2023 14:58:19 -0800 Subject: benchmark: update COMPARISON --- benchmark/COMPARISON | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index c26c6adb..fd41282e 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,4 +1,4 @@ CPU | MHz | threads | singlecore | multicore | afl-*-config | ====================================================|=======|=========|============|===========|==============| -Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 174386 | 1112585 | both | -Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 133706 | 1169989 | both | +Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | +Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 120064 | 1168943 | both | -- cgit 1.4.1