From 16993bba8fa359b093d44fe395044bfd392c19f3 Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sat, 11 Nov 2023 01:45:13 +0000 Subject: benchmark: Add support for COMPARISON file --- benchmark/COMPARISON | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index 55ab94b4..d8750e34 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,4 +1,4 @@ -CPU | Mz | exec/s | afl-*-config | -========================================|======|========|==============| -CPU 12th Gen Intel(R) Core(TM) i7-1270P | 4200 | 12750 | both | -AMD EPYC 7282 16-Core Processor | 3190 | 10060 | both | +CPU | MHz | singlecore | multicore | afl-*-config | +===============================|=======|============|===========|==============| +Apple Mac Studio M2 Ultra 2023 | 3500 | 174386 | 1112585 | both | +Intel(R) Core(TM) i9-9900K CPU | 4999 | 133706 | 1169989 | both | -- cgit 1.4.1 From 8b79d9b4d5efdf4d0e30a323dd3a169f9a7801c5 Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sun, 12 Nov 2023 07:40:58 -0800 Subject: benchmark: show the number of cores used in COMPARISON --- benchmark/COMPARISON | 8 ++++---- benchmark/benchmark.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index d8750e34..03a197cd 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,4 +1,4 @@ -CPU | MHz | singlecore | multicore | afl-*-config | -===============================|=======|============|===========|==============| -Apple Mac Studio M2 Ultra 2023 | 3500 | 174386 | 1112585 | both | -Intel(R) Core(TM) i9-9900K CPU | 4999 | 133706 | 1169989 | both | +CPU | MHz | threads | singlecore | multicore | afl-*-config | +===============================|=======|=========|============|===========|==============| +Apple Mac Studio M2 Ultra 2023 | 3500 | 16 | 174386 | 1112585 | both | +Intel(R) Core(TM) i9-9900K CPU | 4995 | 16 | 133706 | 1169989 | both | diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 6fde621b..2601ef0d 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -195,6 +195,7 @@ async def save_benchmark_results() -> None: print(blue(f" [*] Results have been written to {jsonfile.name}")) with open("COMPARISON", "a") as comparisonfile: described_config = await describe_afl_config() + aflconfig = described_config.ljust(12) if results.hardware is None: return cpu_model = results.hardware.cpu_model.ljust(42) @@ -205,9 +206,10 @@ async def save_benchmark_results() -> None: return single = str(round(results.targets["test-instr-persist-shmem"]["singlecore"].total_execs_per_sec)).ljust(10) multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].total_execs_per_sec)).ljust(9) + cores = str(args.fuzzers).ljust(7) if len(cpu_model) > 30: cpu_model = cpu_model[:30] - comparisonfile.write(f"{cpu_model} | {cpu_mhz} | {single} | {multi} | {described_config.ljust(12)} |\n") + comparisonfile.write(f"{cpu_model} | {cpu_mhz} | {cores} | {single} | {multi} | {aflconfig} |\n") with open("COMPARISON", "r") as comparisonfile: print(comparisonfile.read()) -- cgit 1.4.1 From 26045831a23ceef70834c7a7be4d17d436a4cf8e Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sun, 12 Nov 2023 11:52:17 -0800 Subject: benchmark: use afl's execs/s; increase CPU model width --- benchmark/COMPARISON | 8 ++++---- benchmark/benchmark.py | 9 +++------ 2 files changed, 7 insertions(+), 10 deletions(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index 03a197cd..c26c6adb 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,4 +1,4 @@ -CPU | MHz | threads | singlecore | multicore | afl-*-config | -===============================|=======|=========|============|===========|==============| -Apple Mac Studio M2 Ultra 2023 | 3500 | 16 | 174386 | 1112585 | both | -Intel(R) Core(TM) i9-9900K CPU | 4995 | 16 | 133706 | 1169989 | both | +CPU | MHz | threads | singlecore | multicore | afl-*-config | +====================================================|=======|=========|============|===========|==============| +Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 174386 | 1112585 | both | +Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 133706 | 1169989 | both | diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 5b363e16..1510acb6 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -198,17 +198,15 @@ async def save_benchmark_results() -> None: aflconfig = described_config.ljust(12) if results.hardware is None: return - cpu_model = results.hardware.cpu_model.ljust(42) + cpu_model = results.hardware.cpu_model.ljust(51) cpu_mhz = str(round(results.hardware.cpu_fastest_core_mhz)).ljust(5) if "test-instr-persist-shmem" in results.targets and "multicore" in results.targets["test-instr-persist-shmem"]: if results.targets["test-instr-persist-shmem"]["singlecore"] is None or \ results.targets["test-instr-persist-shmem"]["multicore"] is None: return - single = str(round(results.targets["test-instr-persist-shmem"]["singlecore"].total_execs_per_sec)).ljust(10) - multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].total_execs_per_sec)).ljust(9) + single = str(round(results.targets["test-instr-persist-shmem"]["singlecore"].afl_execs_per_sec)).ljust(10) + multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].afl_execs_per_sec)).ljust(9) cores = str(args.fuzzers).ljust(7) - if len(cpu_model) > 30: - cpu_model = cpu_model[:30] comparisonfile.write(f"{cpu_model} | {cpu_mhz} | {cores} | {single} | {multi} | {aflconfig} |\n") with open("COMPARISON", "r") as comparisonfile: print(comparisonfile.read()) @@ -281,7 +279,6 @@ async def main() -> None: results.targets[binary][mode.name] = run print(f" [*] Average AFL execs/sec for this test across all runs was: {green(avg_afl_execs_per_sec)}") - print(f" [*] Average total execs/sec for this test across all runs was: {green(total_execs_per_sec)}") if (((max(afl_execs_per_sec) - min(afl_execs_per_sec)) / avg_afl_execs_per_sec) * 100) > 15: print(yellow(" [*] The difference between your slowest and fastest runs was >15%, maybe try again?")) -- cgit 1.4.1 From 75a3af8a23f2fd1488ee7cb6e5fca00289031c6c Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sun, 19 Nov 2023 14:58:19 -0800 Subject: benchmark: update COMPARISON --- benchmark/COMPARISON | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index c26c6adb..fd41282e 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,4 +1,4 @@ CPU | MHz | threads | singlecore | multicore | afl-*-config | ====================================================|=======|=========|============|===========|==============| -Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 174386 | 1112585 | both | -Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 133706 | 1169989 | both | +Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | +Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 120064 | 1168943 | both | -- cgit 1.4.1 From 07352a932ba47fdd69468451750be6b1f5958c55 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 20 Nov 2023 09:31:43 +0100 Subject: add benchmark --- benchmark/COMPARISON | 1 + benchmark/benchmark-results.jsonl | 1 + 2 files changed, 2 insertions(+) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index fd41282e..ba82baf2 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -2,3 +2,4 @@ CPU | MHz | threads | singleco ====================================================|=======|=========|============|===========|==============| Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 120064 | 1168943 | both | +12th Gen Intel(R) Core(TM) i7-1270P | 4761 | 16 | 149778 | 641219 | both | diff --git a/benchmark/benchmark-results.jsonl b/benchmark/benchmark-results.jsonl index f3dd60be..2c327b56 100644 --- a/benchmark/benchmark-results.jsonl +++ b/benchmark/benchmark-results.jsonl @@ -413,3 +413,4 @@ {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.599, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 476949.52, "execs_total": 4772500, "fuzzers_used": 190}}}} {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3437.101, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 474259.76, "execs_total": 4745505, "fuzzers_used": 191}}}} {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.17, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 479848.23, "execs_total": 4801111, "fuzzers_used": 192}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Debian clang version 17.0.4 (++20231031083102+309d55140c46-1~exp1~20231031083155.63)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4761.063, "cpu_model": "12th Gen Intel(R) Core(TM) i7-1270P", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 641219.02, "execs_total": 19251242, "fuzzers_used": 16}, "singlecore": {"execs_per_sec": 149778.22, "execs_total": 4493796, "fuzzers_used": 1}}}} -- cgit 1.4.1 From aabbdac86d6215833391a54fa7d3a474ad41e3fd Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 20 Nov 2023 09:56:09 +0100 Subject: add benchmarks --- benchmark/COMPARISON | 4 +++- benchmark/README.md | 8 ++++++++ benchmark/benchmark-results.jsonl | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index ba82baf2..a8de1a60 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,5 +1,7 @@ CPU | MHz | threads | singlecore | multicore | afl-*-config | ====================================================|=======|=========|============|===========|==============| -Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 120064 | 1168943 | both | +Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | +AMD EPYC 7282 16-Core Processor | 3194 | 32 | 87199 | 769001 | both | +AMD Ryzen 5 PRO 4650G with Radeon Graphics | 3700 | 12 | 95356 | 704840 | both | 12th Gen Intel(R) Core(TM) i7-1270P | 4761 | 16 | 149778 | 641219 | both | diff --git a/benchmark/README.md b/benchmark/README.md index e37abad2..c7d75e42 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -9,6 +9,14 @@ To achieve this, we use a sample program ("test-instr.c") where each path is equally likely, supply it a single seed, and tell AFL to exit after one run of deterministic mutations against that seed. +**Note that this is not a real-world scenario!** +Because the target does basically nothing this is rather a stress test on +Kernel I/O / context switching. +For this reason you will not see a difference if you run the multicore test +with 20 or 40 threads - or even see the performance decline the more threads +(`-f` parameter) you use. In a real-world scenario you can expect to gain +exec/s until 40-60 threads (if you have that many available on your CPU). + Usage example: ``` diff --git a/benchmark/benchmark-results.jsonl b/benchmark/benchmark-results.jsonl index 2c327b56..9b8ef038 100644 --- a/benchmark/benchmark-results.jsonl +++ b/benchmark/benchmark-results.jsonl @@ -414,3 +414,5 @@ {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3437.101, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 474259.76, "execs_total": 4745505, "fuzzers_used": 191}}}} {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.17, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 479848.23, "execs_total": 4801111, "fuzzers_used": 192}}}} {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Debian clang version 17.0.4 (++20231031083102+309d55140c46-1~exp1~20231031083155.63)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4761.063, "cpu_model": "12th Gen Intel(R) Core(TM) i7-1270P", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 641219.02, "execs_total": 19251242, "fuzzers_used": 16}, "singlecore": {"execs_per_sec": 149778.22, "execs_total": 4493796, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Ubuntu clang version 17.0.2 (++20231003073128+b2417f51dbbd-1~exp1~20231003073233.51)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3193.942, "cpu_model": "AMD EPYC 7282 16-Core Processor", "cpu_threads": 64}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 769000.8, "execs_total": 23084516, "fuzzers_used": 32}, "singlecore": {"execs_per_sec": 87198.85, "execs_total": 2616227, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.08a", "comment": "", "compiler": "Ubuntu clang version 14.0.0-1ubuntu1.1", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3700.0, "cpu_model": "AMD Ryzen 5 PRO 4650G with Radeon Graphics", "cpu_threads": 12}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 704840.16, "execs_total": 21163992, "fuzzers_used": 12}, "singlecore": {"execs_per_sec": 95356.14, "execs_total": 2862114, "fuzzers_used": 1}}}} -- cgit 1.4.1 From b0cb2f7e83d3f0273761a3adf0cb6bcbfd5395c6 Mon Sep 17 00:00:00 2001 From: vH Date: Sun, 3 Dec 2023 12:54:44 +0100 Subject: ryzen 5950 benchmark --- benchmark/COMPARISON | 1 + benchmark/benchmark-results.jsonl | 1 + 2 files changed, 2 insertions(+) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index a8de1a60..801b58d8 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -5,3 +5,4 @@ Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 AMD EPYC 7282 16-Core Processor | 3194 | 32 | 87199 | 769001 | both | AMD Ryzen 5 PRO 4650G with Radeon Graphics | 3700 | 12 | 95356 | 704840 | both | 12th Gen Intel(R) Core(TM) i7-1270P | 4761 | 16 | 149778 | 641219 | both | +AMD Ryzen 9 5950X 16-Core Processor | 4792 | 32 | 161690 | 2339763 | both | diff --git a/benchmark/benchmark-results.jsonl b/benchmark/benchmark-results.jsonl index 2dd9f406..ac800d65 100644 --- a/benchmark/benchmark-results.jsonl +++ b/benchmark/benchmark-results.jsonl @@ -417,3 +417,4 @@ {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Ubuntu clang version 17.0.2 (++20231003073128+b2417f51dbbd-1~exp1~20231003073233.51)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3193.942, "cpu_model": "AMD EPYC 7282 16-Core Processor", "cpu_threads": 64}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 769000.8, "execs_total": 23084516, "fuzzers_used": 32}, "singlecore": {"execs_per_sec": 87198.85, "execs_total": 2616227, "fuzzers_used": 1}}}} {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.08a", "comment": "", "compiler": "Ubuntu clang version 14.0.0-1ubuntu1.1", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3700.0, "cpu_model": "AMD Ryzen 5 PRO 4650G with Radeon Graphics", "cpu_threads": 12}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 704840.16, "execs_total": 21163992, "fuzzers_used": 12}, "singlecore": {"execs_per_sec": 95356.14, "execs_total": 2862114, "fuzzers_used": 1}}}} {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Debian clang version 14.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 2400.0, "cpu_model": "Raspberry Pi 5", "cpu_threads": 4}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 101114.23, "execs_total": 3036637, "fuzzers_used": 4}, "singlecore": {"execs_per_sec": 25786.11, "execs_total": 774460, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.07a", "comment": "", "compiler": "Debian clang version 17.0.0 (++20230417071830+ae77aceba5ad-1~exp1~20230417071935.630)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4792.073, "cpu_model": "AMD Ryzen 9 5950X 16-Core Processor", "cpu_threads": 32}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 2339762.91, "execs_total": 70253164, "fuzzers_used": 32}, "singlecore": {"execs_per_sec": 161690.07, "execs_total": 4851838, "fuzzers_used": 1}}}} -- cgit 1.4.1 From 477a5176281a907c9c54d4588c88ea1ed93e72e2 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 3 Dec 2023 13:01:34 +0100 Subject: add missing raspery5 --- benchmark/COMPARISON | 1 + 1 file changed, 1 insertion(+) (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index 801b58d8..f3ac3687 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -5,4 +5,5 @@ Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 AMD EPYC 7282 16-Core Processor | 3194 | 32 | 87199 | 769001 | both | AMD Ryzen 5 PRO 4650G with Radeon Graphics | 3700 | 12 | 95356 | 704840 | both | 12th Gen Intel(R) Core(TM) i7-1270P | 4761 | 16 | 149778 | 641219 | both | +Raspberry Pi 5 | 2400 | 4 | 774460 | 3036637 | both AMD Ryzen 9 5950X 16-Core Processor | 4792 | 32 | 161690 | 2339763 | both | -- cgit 1.4.1 From 01e0d4aa1c9e856124491d1f23deea0ae443d8ea Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 3 Dec 2023 13:12:22 +0100 Subject: comparison -> comparison.md --- benchmark/COMPARISON | 9 --------- benchmark/README.md | 6 +++--- benchmark/benchmark.py | 8 ++++---- 3 files changed, 7 insertions(+), 16 deletions(-) delete mode 100644 benchmark/COMPARISON (limited to 'benchmark/COMPARISON') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON deleted file mode 100644 index f3ac3687..00000000 --- a/benchmark/COMPARISON +++ /dev/null @@ -1,9 +0,0 @@ -CPU | MHz | threads | singlecore | multicore | afl-*-config | -====================================================|=======|=========|============|===========|==============| -Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 120064 | 1168943 | both | -Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | -AMD EPYC 7282 16-Core Processor | 3194 | 32 | 87199 | 769001 | both | -AMD Ryzen 5 PRO 4650G with Radeon Graphics | 3700 | 12 | 95356 | 704840 | both | -12th Gen Intel(R) Core(TM) i7-1270P | 4761 | 16 | 149778 | 641219 | both | -Raspberry Pi 5 | 2400 | 4 | 774460 | 3036637 | both -AMD Ryzen 9 5950X 16-Core Processor | 4792 | 32 | 161690 | 2339763 | both | diff --git a/benchmark/README.md b/benchmark/README.md index c7d75e42..12f4763e 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -32,7 +32,7 @@ python3 benchmark.py [*] multicore test-instr-persist-shmem run 2 of 2, execs/s: 1175584.09 [*] Average execs/sec for this test across all runs was: 1177703.38 [*] Results have been written to the benchmark-results.jsonl file. - [*] Results have been written to the COMPARISON file. + [*] Results have been written to the COMPARISON.md file. ``` By default, the script will use a number of parallel fuzzers equal to your @@ -43,8 +43,8 @@ The script will use multicore fuzzing instead of singlecore by default (change with `--mode singlecore`) and use a persistent-mode shared memory harness for optimal speed (change with `--target test-instr`). -Feel free to submit the resulting line for your CPU added to the COMPARISON -file back to aflplusplus in a pull request. +Feel free to submit the resulting line for your CPU added to the COMPARISON.md +and benchmark-results.jsonl files back to AFL++ in a pull request. Each run writes results to [benchmark-results.jsonl](benchmark-results.jsonl) in [JSON Lines](https://jsonlines.org/) format, ready to be pulled in to other diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index b3d55f21..0685cedd 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -186,14 +186,14 @@ async def save_benchmark_results() -> None: json.dump(asdict(results), jsonfile, sort_keys=True) jsonfile.write("\n") print(blue(f" [*] Results have been written to the {jsonfile.name} file.")) - with open("COMPARISON", "r+") as comparisonfile: + with open("COMPARISON.md", "r+") as comparisonfile: described_config = await describe_afl_config() aflconfig = described_config.ljust(12) if results.hardware is None: return cpu_model = results.hardware.cpu_model.ljust(51) if cpu_model in comparisonfile.read(): - print(blue(f" [*] Results have not been written to the COMPARISON file; this CPU is already present.")) + print(blue(f" [*] Results have not been written to the COMPARISON.md file; this CPU is already present.")) return cpu_mhz = str(round(results.hardware.cpu_fastest_core_mhz)).ljust(5) if not "test-instr-persist-shmem" in results.targets or \ @@ -206,8 +206,8 @@ async def save_benchmark_results() -> None: multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].execs_per_sec)).ljust(9) cores = str(args.fuzzers).ljust(7) comparisonfile.write(f"{cpu_model} | {cpu_mhz} | {cores} | {single} | {multi} | {aflconfig} |\n") - print(blue(f" [*] Results have been written to the COMPARISON file.")) - with open("COMPARISON", "r") as comparisonfile: + print(blue(f" [*] Results have been written to the COMPARISON.md file.")) + with open("COMPARISON.md", "r") as comparisonfile: print(comparisonfile.read()) -- cgit 1.4.1