From b9db6b1254c9bf3a47c171bb96468628e9bd00f2 Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Mon, 2 Oct 2023 03:23:09 -0700 Subject: benchmark: add a README, lower default runs from 5 to 3 --- benchmark/README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 benchmark/README.md (limited to 'benchmark/README.md') diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..66f7f59e --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,46 @@ +# American Fuzzy Lop plus plus (AFL++) + +## benchmarking + +This directory contains benchmarking tools that allow you to compare one machine +with another in terms of raw ability to execute a fuzzing target repeatedly. + +To achieve this, we use a sample program ("test-instr.c") where each path is +equally likely, supply it a single seed, and tell AFL to exit after one run of +deterministic mutations against that seed. + +Usage: + +``` +cd aflplusplus/benchmark +python3 benchmark.py + [*] Using 16 fuzzers for multicore fuzzing (use --fuzzers to override) + [*] Ready, starting benchmark... + [*] Compiling the test-instr-persist-shmem fuzzing harness for the benchmark to use. + [*] multicore test-instr-persist-shmem run 1 of 3, execs/s: 846065.81 + [*] multicore test-instr-persist-shmem run 2 of 3, execs/s: 849694.03 + [*] multicore test-instr-persist-shmem run 3 of 3, execs/s: 850757.52 + [*] Average AFL execs/sec for this test across all runs was: 848839.12 + [*] Average total execs/sec for this test across all runs was: 833138.28 + [*] Results have been written to benchmark-results.jsonl +``` + +By default, the script will use a number of parallel fuzzers equal to your +available CPUs/threads (change with `--fuzzers`), and will perform each test +three times and average the result (change with `--runs`). + +The script will use multicore fuzzing instead of singlecore by default (change +with `--mode singlecore`) and use a persistent-mode shared memory harness for +optimal speed (change with `--target test-instr`). + +Each run writes results to [benchmark-results.jsonl](benchmark-results.jsonl) +in [JSON Lines](https://jsonlines.org/) format, ready to be pulled in to other +tools such as [jq -cs](https://jqlang.github.io/jq/) or +[pandas](https://pandas.pydata.org/) for analysis. + +## Data analysis + +There is sample data in [benchmark-results.jsonl](benchmark-results.jsonl), and +a Jupyter notebook for exploring the results and suggesting their meaning at +[benchmark.ipynb](benchmark.ipynb). + -- cgit 1.4.1 From f2cbcdf3ff7349ab505e1fcebc3242c9252f2176 Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Sun, 19 Nov 2023 15:10:23 -0800 Subject: benchmark: update README --- benchmark/README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'benchmark/README.md') diff --git a/benchmark/README.md b/benchmark/README.md index 66f7f59e..e37abad2 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -9,20 +9,22 @@ To achieve this, we use a sample program ("test-instr.c") where each path is equally likely, supply it a single seed, and tell AFL to exit after one run of deterministic mutations against that seed. -Usage: +Usage example: ``` cd aflplusplus/benchmark python3 benchmark.py - [*] Using 16 fuzzers for multicore fuzzing (use --fuzzers to override) [*] Ready, starting benchmark... [*] Compiling the test-instr-persist-shmem fuzzing harness for the benchmark to use. - [*] multicore test-instr-persist-shmem run 1 of 3, execs/s: 846065.81 - [*] multicore test-instr-persist-shmem run 2 of 3, execs/s: 849694.03 - [*] multicore test-instr-persist-shmem run 3 of 3, execs/s: 850757.52 - [*] Average AFL execs/sec for this test across all runs was: 848839.12 - [*] Average total execs/sec for this test across all runs was: 833138.28 - [*] Results have been written to benchmark-results.jsonl + [*] singlecore test-instr-persist-shmem run 1 of 2, execs/s: 124883.62 + [*] singlecore test-instr-persist-shmem run 2 of 2, execs/s: 126704.93 + [*] Average execs/sec for this test across all runs was: 125794.28 + [*] Using 16 fuzzers for multicore fuzzing (use --fuzzers to override). + [*] multicore test-instr-persist-shmem run 1 of 2, execs/s: 1179822.66 + [*] multicore test-instr-persist-shmem run 2 of 2, execs/s: 1175584.09 + [*] Average execs/sec for this test across all runs was: 1177703.38 + [*] Results have been written to the benchmark-results.jsonl file. + [*] Results have been written to the COMPARISON file. ``` By default, the script will use a number of parallel fuzzers equal to your @@ -33,6 +35,9 @@ The script will use multicore fuzzing instead of singlecore by default (change with `--mode singlecore`) and use a persistent-mode shared memory harness for optimal speed (change with `--target test-instr`). +Feel free to submit the resulting line for your CPU added to the COMPARISON +file back to aflplusplus in a pull request. + Each run writes results to [benchmark-results.jsonl](benchmark-results.jsonl) in [JSON Lines](https://jsonlines.org/) format, ready to be pulled in to other tools such as [jq -cs](https://jqlang.github.io/jq/) or -- cgit 1.4.1 From aabbdac86d6215833391a54fa7d3a474ad41e3fd Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 20 Nov 2023 09:56:09 +0100 Subject: add benchmarks --- benchmark/COMPARISON | 4 +++- benchmark/README.md | 8 ++++++++ benchmark/benchmark-results.jsonl | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) (limited to 'benchmark/README.md') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON index ba82baf2..a8de1a60 100644 --- a/benchmark/COMPARISON +++ b/benchmark/COMPARISON @@ -1,5 +1,7 @@ CPU | MHz | threads | singlecore | multicore | afl-*-config | ====================================================|=======|=========|============|===========|==============| -Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 120064 | 1168943 | both | +Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | +AMD EPYC 7282 16-Core Processor | 3194 | 32 | 87199 | 769001 | both | +AMD Ryzen 5 PRO 4650G with Radeon Graphics | 3700 | 12 | 95356 | 704840 | both | 12th Gen Intel(R) Core(TM) i7-1270P | 4761 | 16 | 149778 | 641219 | both | diff --git a/benchmark/README.md b/benchmark/README.md index e37abad2..c7d75e42 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -9,6 +9,14 @@ To achieve this, we use a sample program ("test-instr.c") where each path is equally likely, supply it a single seed, and tell AFL to exit after one run of deterministic mutations against that seed. +**Note that this is not a real-world scenario!** +Because the target does basically nothing this is rather a stress test on +Kernel I/O / context switching. +For this reason you will not see a difference if you run the multicore test +with 20 or 40 threads - or even see the performance decline the more threads +(`-f` parameter) you use. In a real-world scenario you can expect to gain +exec/s until 40-60 threads (if you have that many available on your CPU). + Usage example: ``` diff --git a/benchmark/benchmark-results.jsonl b/benchmark/benchmark-results.jsonl index 2c327b56..9b8ef038 100644 --- a/benchmark/benchmark-results.jsonl +++ b/benchmark/benchmark-results.jsonl @@ -414,3 +414,5 @@ {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3437.101, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 474259.76, "execs_total": 4745505, "fuzzers_used": 191}}}} {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.17, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 479848.23, "execs_total": 4801111, "fuzzers_used": 192}}}} {"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Debian clang version 17.0.4 (++20231031083102+309d55140c46-1~exp1~20231031083155.63)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4761.063, "cpu_model": "12th Gen Intel(R) Core(TM) i7-1270P", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 641219.02, "execs_total": 19251242, "fuzzers_used": 16}, "singlecore": {"execs_per_sec": 149778.22, "execs_total": 4493796, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Ubuntu clang version 17.0.2 (++20231003073128+b2417f51dbbd-1~exp1~20231003073233.51)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3193.942, "cpu_model": "AMD EPYC 7282 16-Core Processor", "cpu_threads": 64}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 769000.8, "execs_total": 23084516, "fuzzers_used": 32}, "singlecore": {"execs_per_sec": 87198.85, "execs_total": 2616227, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.08a", "comment": "", "compiler": "Ubuntu clang version 14.0.0-1ubuntu1.1", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3700.0, "cpu_model": "AMD Ryzen 5 PRO 4650G with Radeon Graphics", "cpu_threads": 12}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 704840.16, "execs_total": 21163992, "fuzzers_used": 12}, "singlecore": {"execs_per_sec": 95356.14, "execs_total": 2862114, "fuzzers_used": 1}}}} -- cgit 1.4.1 From 01e0d4aa1c9e856124491d1f23deea0ae443d8ea Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 3 Dec 2023 13:12:22 +0100 Subject: comparison -> comparison.md --- benchmark/COMPARISON | 9 --------- benchmark/README.md | 6 +++--- benchmark/benchmark.py | 8 ++++---- 3 files changed, 7 insertions(+), 16 deletions(-) delete mode 100644 benchmark/COMPARISON (limited to 'benchmark/README.md') diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON deleted file mode 100644 index f3ac3687..00000000 --- a/benchmark/COMPARISON +++ /dev/null @@ -1,9 +0,0 @@ -CPU | MHz | threads | singlecore | multicore | afl-*-config | -====================================================|=======|=========|============|===========|==============| -Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 120064 | 1168943 | both | -Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | -AMD EPYC 7282 16-Core Processor | 3194 | 32 | 87199 | 769001 | both | -AMD Ryzen 5 PRO 4650G with Radeon Graphics | 3700 | 12 | 95356 | 704840 | both | -12th Gen Intel(R) Core(TM) i7-1270P | 4761 | 16 | 149778 | 641219 | both | -Raspberry Pi 5 | 2400 | 4 | 774460 | 3036637 | both -AMD Ryzen 9 5950X 16-Core Processor | 4792 | 32 | 161690 | 2339763 | both | diff --git a/benchmark/README.md b/benchmark/README.md index c7d75e42..12f4763e 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -32,7 +32,7 @@ python3 benchmark.py [*] multicore test-instr-persist-shmem run 2 of 2, execs/s: 1175584.09 [*] Average execs/sec for this test across all runs was: 1177703.38 [*] Results have been written to the benchmark-results.jsonl file. - [*] Results have been written to the COMPARISON file. + [*] Results have been written to the COMPARISON.md file. ``` By default, the script will use a number of parallel fuzzers equal to your @@ -43,8 +43,8 @@ The script will use multicore fuzzing instead of singlecore by default (change with `--mode singlecore`) and use a persistent-mode shared memory harness for optimal speed (change with `--target test-instr`). -Feel free to submit the resulting line for your CPU added to the COMPARISON -file back to aflplusplus in a pull request. +Feel free to submit the resulting line for your CPU added to the COMPARISON.md +and benchmark-results.jsonl files back to AFL++ in a pull request. Each run writes results to [benchmark-results.jsonl](benchmark-results.jsonl) in [JSON Lines](https://jsonlines.org/) format, ready to be pulled in to other diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index b3d55f21..0685cedd 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -186,14 +186,14 @@ async def save_benchmark_results() -> None: json.dump(asdict(results), jsonfile, sort_keys=True) jsonfile.write("\n") print(blue(f" [*] Results have been written to the {jsonfile.name} file.")) - with open("COMPARISON", "r+") as comparisonfile: + with open("COMPARISON.md", "r+") as comparisonfile: described_config = await describe_afl_config() aflconfig = described_config.ljust(12) if results.hardware is None: return cpu_model = results.hardware.cpu_model.ljust(51) if cpu_model in comparisonfile.read(): - print(blue(f" [*] Results have not been written to the COMPARISON file; this CPU is already present.")) + print(blue(f" [*] Results have not been written to the COMPARISON.md file; this CPU is already present.")) return cpu_mhz = str(round(results.hardware.cpu_fastest_core_mhz)).ljust(5) if not "test-instr-persist-shmem" in results.targets or \ @@ -206,8 +206,8 @@ async def save_benchmark_results() -> None: multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].execs_per_sec)).ljust(9) cores = str(args.fuzzers).ljust(7) comparisonfile.write(f"{cpu_model} | {cpu_mhz} | {cores} | {single} | {multi} | {aflconfig} |\n") - print(blue(f" [*] Results have been written to the COMPARISON file.")) - with open("COMPARISON", "r") as comparisonfile: + print(blue(f" [*] Results have been written to the COMPARISON.md file.")) + with open("COMPARISON.md", "r") as comparisonfile: print(comparisonfile.read()) -- cgit 1.4.1