From b9db6b1254c9bf3a47c171bb96468628e9bd00f2 Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Mon, 2 Oct 2023 03:23:09 -0700 Subject: benchmark: add a README, lower default runs from 5 to 3 --- benchmark/README.md | 46 + benchmark/benchmark.ipynb | 3502 +-------------------------------------------- benchmark/benchmark.py | 2 +- 3 files changed, 110 insertions(+), 3440 deletions(-) create mode 100644 benchmark/README.md diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..66f7f59e --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,46 @@ +# American Fuzzy Lop plus plus (AFL++) + +## benchmarking + +This directory contains benchmarking tools that allow you to compare one machine +with another in terms of raw ability to execute a fuzzing target repeatedly. + +To achieve this, we use a sample program ("test-instr.c") where each path is +equally likely, supply it a single seed, and tell AFL to exit after one run of +deterministic mutations against that seed. + +Usage: + +``` +cd aflplusplus/benchmark +python3 benchmark.py + [*] Using 16 fuzzers for multicore fuzzing (use --fuzzers to override) + [*] Ready, starting benchmark... + [*] Compiling the test-instr-persist-shmem fuzzing harness for the benchmark to use. + [*] multicore test-instr-persist-shmem run 1 of 3, execs/s: 846065.81 + [*] multicore test-instr-persist-shmem run 2 of 3, execs/s: 849694.03 + [*] multicore test-instr-persist-shmem run 3 of 3, execs/s: 850757.52 + [*] Average AFL execs/sec for this test across all runs was: 848839.12 + [*] Average total execs/sec for this test across all runs was: 833138.28 + [*] Results have been written to benchmark-results.jsonl +``` + +By default, the script will use a number of parallel fuzzers equal to your +available CPUs/threads (change with `--fuzzers`), and will perform each test +three times and average the result (change with `--runs`). + +The script will use multicore fuzzing instead of singlecore by default (change +with `--mode singlecore`) and use a persistent-mode shared memory harness for +optimal speed (change with `--target test-instr`). + +Each run writes results to [benchmark-results.jsonl](benchmark-results.jsonl) +in [JSON Lines](https://jsonlines.org/) format, ready to be pulled in to other +tools such as [jq -cs](https://jqlang.github.io/jq/) or +[pandas](https://pandas.pydata.org/) for analysis. + +## Data analysis + +There is sample data in [benchmark-results.jsonl](benchmark-results.jsonl), and +a Jupyter notebook for exploring the results and suggesting their meaning at +[benchmark.ipynb](benchmark.ipynb). + diff --git a/benchmark/benchmark.ipynb b/benchmark/benchmark.ipynb index b1ef6b0f..7ff90fcd 100644 --- a/benchmark/benchmark.ipynb +++ b/benchmark/benchmark.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 502, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -20,14 +20,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Translate the JSON Lines entries into a single pandas DataFrame.\n", + "### Translate the JSON Lines entries into a single pandas DataFrame\n", "\n", "We have JSON Lines in [benchmark-results.jsonl](benchmark-results.jsonl) that look like this:" ] }, { "cell_type": "code", - "execution_count": 503, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 504, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -371,7 +371,7 @@ "[5 rows x 37 columns]" ] }, - "execution_count": 504, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -404,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 505, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -418,1647 +418,65 @@ }, { "cell_type": "code", - "execution_count": 506, + "execution_count": 16, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
execs_per_secparallel_fuzzersafl_persistent_configafl_system_configlabel
108135613.261.0TrueTrueSinglecore: Persistent mode/shared memory + ke...
117135613.2610.0TrueTrueSinglecore: Persistent mode/shared memory + ke...
\n", - "
" - ], - "text/plain": [ - " execs_per_sec parallel_fuzzers afl_persistent_config \\\n", - "108 135613.26 1.0 True \n", - "117 135613.26 10.0 True \n", - "\n", - " afl_system_config label \n", - "108 True Singlecore: Persistent mode/shared memory + ke... \n", - "117 True Singlecore: Persistent mode/shared memory + ke... " - ] - }, - "execution_count": 506, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "def build_graphdf_from_query(query: pd.DataFrame):\n", " \"\"\"Build a table suitable for graphing from a subset of the dataframe.\"\"\"\n", " graphdata = []\n", " max_fuzzers = int(query[[\"targets.test-instr-persist-shmem.multicore.fuzzers_used\", \"targets.test-instr.multicore.fuzzers_used\"]].max(axis=1).max(axis=0))\n", " for _, row in query.iterrows():\n", - " if row[\"targets.test-instr-persist-shmem.multicore.total_execs_per_sec\"] > 0:\n", - " execs_per_sec = row[\"targets.test-instr-persist-shmem.multicore.total_execs_per_sec\"]\n", - " afl_execs_per_sec = row[\"targets.test-instr-persist-shmem.multicore.afl_execs_per_sec\"]\n", - " parallel_fuzzers = row[\"targets.test-instr-persist-shmem.multicore.fuzzers_used\"]\n", - " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", - " afl_system_config = row[\"config.afl_system_config\"]\n", - " label = \"shmem-multicore\"\n", - " if afl_persistent_config:\n", - " label += \"+persist-conf\"\n", - " if afl_system_config:\n", - " label += \"+system-conf\"\n", - " if label == \"shmem-multicore+persist-conf+system-conf\":\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory + kernel config\"})\n", - " graphdata.append({\"execs_per_sec\": afl_execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: afl_execs: Persistent mode/shared memory + kernel config\"})\n", - " if label == \"shmem-multicore\":\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory without kernel config\"})\n", - " if row[\"targets.test-instr.multicore.total_execs_per_sec\"] > 0:\n", - " execs_per_sec = row[\"targets.test-instr.multicore.total_execs_per_sec\"]\n", - " parallel_fuzzers = row[\"targets.test-instr.multicore.fuzzers_used\"]\n", - " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", - " afl_system_config = row[\"config.afl_system_config\"]\n", - " label = \"base-multicore\"\n", - " if afl_persistent_config:\n", - " label += \"+persist-conf\"\n", - " if afl_system_config:\n", - " label += \"+system-conf\"\n", + " for target in [\"test-instr-persist-shmem\", \"test-instr\"]:\n", + " for mode in [\"multicore\", \"singlecore\"]:\n", + " label = \"\"\n", + " if not row[f\"targets.{target}.{mode}.total_execs_per_sec\"] > 0:\n", + " continue\n", + " execs_per_sec = row[f\"targets.{target}.{mode}.total_execs_per_sec\"]\n", + " afl_execs_per_sec = row[f\"targets.{target}.{mode}.afl_execs_per_sec\"]\n", + " parallel_fuzzers = row[f\"targets.{target}.{mode}.fuzzers_used\"]\n", + " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", + " afl_system_config = row[\"config.afl_system_config\"]\n", + " if target == \"test-instr-persist-shmem\":\n", + " label += \"shmem\"\n", + " else:\n", + " label += \"base\"\n", + " if mode == \"multicore\":\n", + " label += \"-multicore\"\n", + " else:\n", + " label += \"-singlecore\"\n", + " if afl_persistent_config:\n", + " label += \"+persist-conf\"\n", + " if afl_system_config:\n", + " label += \"+system-conf\"\n", + " \n", + " if label == \"shmem-multicore+persist-conf+system-conf\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory + kernel config\"})\n", + " graphdata.append({\"execs_per_sec\": afl_execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: afl_execs: Persistent mode/shared memory + kernel config\"})\n", + " if label == \"shmem-multicore\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory without kernel config\"})\n", " if label == \"base-multicore+persist-conf+system-conf\":\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Non-persistent mode + kernel config\"}) \n", - " if row[\"targets.test-instr-persist-shmem.singlecore.total_execs_per_sec\"] > 0:\n", - " execs_per_sec = row[\"targets.test-instr-persist-shmem.singlecore.total_execs_per_sec\"]\n", - " parallel_fuzzers = row[\"targets.test-instr-persist-shmem.singlecore.fuzzers_used\"]\n", - " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", - " afl_system_config = row[\"config.afl_system_config\"]\n", - " label = \"shmem-singlecore\"\n", - " if afl_persistent_config:\n", - " label += \"+persist-conf\"\n", - " if afl_system_config:\n", - " label += \"+system-conf\"\n", - " if label == \"shmem-singlecore+persist-conf+system-conf\":\n", - " for i in range(1, max_fuzzers + 1):\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Persistent mode/shared memory + kernel config\"})\n", - " if row[\"targets.test-instr.singlecore.total_execs_per_sec\"] > 0:\n", - " execs_per_sec = row[\"targets.test-instr.singlecore.total_execs_per_sec\"]\n", - " parallel_fuzzers = row[\"targets.test-instr.singlecore.fuzzers_used\"]\n", - " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", - " afl_system_config = row[\"config.afl_system_config\"]\n", - " label = \"base-singlecore\"\n", - " if afl_persistent_config:\n", - " label += \"+persist-conf\"\n", - " if afl_system_config:\n", - " label += \"+system-conf\"\n", - " if label == \"base-singlecore+persist-conf+system-conf\":\n", - " for i in range(1, max_fuzzers + 1):\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Non-persistent mode + kernel config\"})\n", - "\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Non-persistent mode + kernel config\"})\n", + " if label == \"shmem-singlecore+persist-conf+system-conf\":\n", + " for i in range(1, max_fuzzers + 1):\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Persistent mode/shared memory + kernel config\"})\n", + " if label == \"base-singlecore+persist-conf+system-conf\":\n", + " for i in range(1, max_fuzzers + 1):\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Non-persistent mode + kernel config\"})\n", " return pd.DataFrame.from_records(graphdata).sort_values(\"label\", ascending=False)\n", "\n", - "graphdf = build_graphdf_from_query(i7)\n", - "graphdf.head(2)" + "graphdf = build_graphdf_from_query(i7)" ] }, { "cell_type": "code", - "execution_count": 507, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "hovertemplate": "Configuration=Multicore: Non-persistent mode + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: Non-persistent mode + kernel config", - "line": { - "color": "#636efa", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: Non-persistent mode + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 11019.3, - 21111.92, - 30568.82, - 38963.07, - 47693.65, - 55718.73, - 64156.79, - 72176.39, - 61257.76, - 67507.14, - 73183.59, - 79167.7, - 85202.55, - 91594.86, - 97682.33, - 103765.38, - 84547.71, - 86611.67, - 88657, - 90134.42, - 91033.28, - 91637.86, - 92747.81, - 93207.38, - 92970.87, - 94162.8, - 94237.96, - 91716.1, - 94072.6, - 95644.79, - 94880.56, - 93460.57, - 94194.83, - 93853.08, - 95147.78, - 92697.06 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: Persistent mode/shared memory + kernel config", - "line": { - "color": "#EF553B", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 133591.26, - 255995.07, - 380246.34, - 490254.72, - 598698.16, - 716786.21, - 824873.02, - 942712.02, - 709716.24, - 779115.44, - 851918.03, - 914375.37, - 990573.31, - 1060551.02, - 1134650.66, - 1203287.99, - 1022498.84, - 1058151.58, - 1076742.64, - 1091743.7, - 1062616.36, - 1081621.57, - 1132929.86, - 1133825.45, - 1114215.27, - 1122210.96, - 1130627.8, - 1135890.71, - 1137390.94, - 1142969.21, - 1149056.35, - 1142131.87, - 1128973.67, - 1116863.53, - 1117913.34, - 1124962.18 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory without kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: Persistent mode/shared memory without kernel config", - "line": { - "color": "#00cc96", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: Persistent mode/shared memory without kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 90851.4, - 176159.32, - 260268.78, - 336355.99, - 413750, - 492578.2, - 565737.17, - 640579.35, - 491284.66, - 540759.63, - 591144.78, - 638938.52, - 687954.18, - 734886.87, - 784210.26, - 834811.24, - 714178.66, - 734804.4, - 740714.93, - 750425.99, - 752625.52, - 760661.34, - 758401.65, - 767985.22, - 785474.61, - 789679.78, - 790483.94, - 798176.69, - 792763.39, - 788972.72, - 789307.69, - 777440.02, - 767985.22, - 775967.55, - 764863.41, - 768616.31 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Multicore: afl_execs: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", - "line": { - "color": "#ab63fa", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 134423.5, - 258490.04, - 383777.45, - 496249.48, - 613089.31, - 730366.19, - 844187.32, - 962846.18, - 997414.74, - 1034757.73, - 1070703.42, - 1104249.08, - 1131176.88, - 1164076.48, - 1198824.47, - 1227578.7, - 1272311.96, - 1295688.8, - 1314398.6, - 1328581.94, - 1342660.66, - 1363930.3, - 1377043.72, - 1375818.24, - 1361687.56, - 1369637.56, - 1375444.16, - 1349599.77, - 1321658.08, - 1301868.24, - 1276904.9, - 1243444.8, - 1243981.21, - 1234425.98, - 1244349.38, - 1250454.58 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Singlecore: Non-persistent mode + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Singlecore: Non-persistent mode + kernel config", - "line": { - "color": "#FFA15A", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Singlecore: Non-persistent mode + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Singlecore: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Singlecore: Persistent mode/shared memory + kernel config", - "line": { - "color": "#19d3f3", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Singlecore: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26 - ], - "yaxis": "y" - } - ], - "layout": { - "height": 400, - "legend": { - "title": { - "text": "Configuration" - }, - "tracegroupgap": 0 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "heatmapgl": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmapgl" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "Fuzzer performance" - }, - "width": 1200, - "xaxis": { - "anchor": "y", - "domain": [ - 0, - 1 - ], - "tickvals": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "title": { - "text": "Number of parallel fuzzers" - } - }, - "yaxis": { - "anchor": "x", - "domain": [ - 0, - 1 - ], - "ticktext": [ - "1x", - "26x", - "51x", - "75x", - "100x", - "125x" - ], - "tickvals": [ - 11019.3, - 284224.18399999995, - 557429.068, - 830633.9519999999, - 1103838.836, - 1377043.72 - ], - "title": { - "text": "Fuzz target executions per second" - } - } - } - }, - "text/html": [ - "
" + "image/svg+xml": [ + "1234567891011121314151617181920212223242526272829303132333435361x26x51x75x100x125xConfigurationMulticore: Non-persistent mode + kernel configMulticore: Persistent mode/shared memory + kernel configMulticore: Persistent mode/shared memory without kernel configMulticore: afl_execs: Persistent mode/shared memory + kernel configSinglecore: Non-persistent mode + kernel configSinglecore: Persistent mode/shared memory + kernel configFuzzer performanceNumber of parallel fuzzersFuzz target executions per second" ] }, "metadata": {}, @@ -2066,7 +484,6 @@ } ], "source": [ - "\n", "import numpy as np\n", "pd.options.plotting.backend = \"plotly\"\n", "\n", @@ -2088,7 +505,8 @@ "# Update the primary Y-axis with custom tick labels\n", "fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n", "fig.update_xaxes(tickvals=list(range(1,36+1)))\n", - "fig.update_layout(width=1200, height=400)\n" + "fig.update_layout(width=1200, height=400)\n", + "fig.show(\"svg\")\n" ] }, { @@ -2100,7 +518,7 @@ }, { "cell_type": "code", - "execution_count": 508, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -2241,7 +659,7 @@ "5.0 135613.26 " ] }, - "execution_count": 508, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -2259,7 +677,7 @@ }, { "cell_type": "code", - "execution_count": 509, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -2318,7 +736,7 @@ "" ] }, - "execution_count": 509, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -2402,7 +820,7 @@ }, { "cell_type": "code", - "execution_count": 521, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -2537,7 +955,7 @@ "149 6.15 " ] }, - "execution_count": 521, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -2549,7 +967,7 @@ }, { "cell_type": "code", - "execution_count": 516, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -2611,7 +1029,7 @@ "153 True Multicore: afl_execs: Persistent mode/shared m... " ] }, - "execution_count": 516, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -2623,1796 +1041,13 @@ }, { "cell_type": "code", - "execution_count": 514, + "execution_count": 22, "metadata": {}, "outputs": [ { "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: Persistent mode/shared memory + kernel config", - "line": { - "color": "#636efa", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 58, - 59, - 60, - 61, - 62, - 63, - 64, - 65, - 66, - 67, - 68, - 69, - 70, - 71, - 72, - 73, - 74, - 75, - 76, - 77, - 78, - 79, - 80, - 81, - 82, - 83, - 84, - 85, - 86, - 87, - 88, - 89, - 90, - 91, - 92, - 93, - 94, - 95, - 96, - 97, - 98, - 99, - 100, - 101, - 102, - 103, - 104, - 105, - 106, - 107, - 108, - 109, - 110, - 111, - 112, - 113, - 114, - 115, - 116, - 117, - 118, - 119, - 120, - 121, - 122, - 123, - 124, - 125, - 126, - 127, - 128, - 129, - 130, - 131, - 132, - 133, - 134, - 135, - 136, - 137, - 138, - 139, - 140, - 141, - 142, - 143, - 144, - 145, - 146, - 147, - 148, - 149, - 150, - 151, - 152, - 153, - 154, - 155, - 156, - 157, - 158, - 159, - 160, - 161, - 162, - 163, - 164, - 165, - 166, - 167, - 168, - 169, - 170, - 171, - 172, - 173, - 174, - 175, - 176, - 177, - 178, - 179, - 180, - 181, - 182, - 183, - 184, - 185, - 186, - 187, - 188, - 189, - 190, - 191, - 192, - 193, - 194, - 195, - 196, - 197, - 198, - 199, - 200 - ], - "xaxis": "x", - "y": [ - 105839.1, - 210819.47, - 316229.21, - 413256.46, - 518632.73, - 619884.69, - 702256.76, - 804131.53, - 847288.04, - 943139.75, - 1024439.07, - 1113578.57, - 1162772.81, - 1265283.48, - 1346295.34, - 1421319.66, - 1389055.03, - 1336294.29, - 1354421.12, - 1341083.87, - 1267487.8, - 1228006.44, - 1164952.24, - 1117569.89, - 1027015.81, - 1011333.83, - 993703.26, - 983824.21, - 969783.14, - 953522.94, - 937704.89, - 914207.81, - 898800.31, - 896437.34, - 895982.76, - 896412.07, - 888119.63, - 874168.22, - 858049.53, - 839870.71, - 830337.88, - 833695.19, - 832246.18, - 831472, - 820819.59, - 809235.61, - 796104.63, - 779748.67, - 651915.77, - 658310.11, - 664906.42, - 670874.88, - 679223.43, - 680460.23, - 683449.31, - 683776.32, - 642820.96, - 656234.7, - 664079.06, - 668815.96, - 677202.95, - 682183.78, - 687798.53, - 695065.41, - 653862.76, - 662255.65, - 667137.25, - 674252.38, - 674769.27, - 676779.76, - 678994.74, - 677707.66, - 643636.07, - 645011.41, - 647860.09, - 650443.61, - 655762.21, - 655894.58, - 659395.36, - 660632.48, - 636619.43, - 636679.37, - 641091.3, - 642134.44, - 644188.04, - 645646.45, - 648190.91, - 645188.95, - 623576.69, - 623521.92, - 627188.81, - 631735.31, - 632169.99, - 632104.98, - 636441.28, - 636413.09, - 627747.11, - 618580.91, - 610289.87, - 602517.1, - 587297.53, - 574782.55, - 559428.41, - 543938.93, - 538120.81, - 531555.69, - 526211.57, - 521796.77, - 510905.75, - 501877.11, - 490630.76, - 479788.36, - 475372.8, - 470776.14, - 466163.29, - 463421.1, - 454522.49, - 446589.03, - 438649.5, - 428977.64, - 424292.46, - 418867.74, - 414980.74, - 412384.2, - 404224.17, - 396310.94, - 389392.67, - 381715.99, - 377825.09, - 374215.78, - 370708.15, - 368402.32, - 361940.65, - 355502.6, - 349589.01, - 342501.52, - 339897.15, - 337989.11, - 335723.22, - 334039.83, - 329127.04, - 323073.5, - 318652.21, - 313198.69, - 311591.71, - 310530.42, - 309390.36, - 307977.56, - 306559.97, - 305066.46, - 302157.29, - 301097.52, - 300138.11, - 299824.88, - 298130.63, - 297838.27, - 296189.21, - 294695.11, - 292518.21, - 291030.08, - 290207.87, - 289828.9, - 289247.99, - 288295.58, - 287447.58, - 286405.31, - 284755.57, - 284046.79, - 283176.27, - 283053.87, - 282178.36, - 281522.15, - 280377.23, - 279885.55, - 278793.08, - 277923.89, - 277069.78, - 277124.24, - 276576.49, - 276020.64, - 275328, - 275029.74, - 274030.79, - 273612.38, - 273285.13, - 273039.33, - 272819.57, - 272960.42, - 272388.01, - 272311.26, - 272115.97, - 272056.42, - 271835.4, - 271397.63, - 271867.2, - 271065.21, - 270797.96, - 270150.29, - 269442.01, - 268674.91 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Multicore: afl_execs: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", - "line": { - "color": "#EF553B", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 58, - 59, - 60, - 61, - 62, - 63, - 64, - 65, - 66, - 67, - 68, - 69, - 70, - 71, - 72, - 73, - 74, - 75, - 76, - 77, - 78, - 79, - 80, - 81, - 82, - 83, - 84, - 85, - 86, - 87, - 88, - 89, - 90, - 91, - 92, - 93, - 94, - 95, - 96, - 97, - 98, - 99, - 100, - 101, - 102, - 103, - 104, - 105, - 106, - 107, - 108, - 109, - 110, - 111, - 112, - 113, - 114, - 115, - 116, - 117, - 118, - 119, - 120, - 121, - 122, - 123, - 124, - 125, - 126, - 127, - 128, - 129, - 130, - 131, - 132, - 133, - 134, - 135, - 136, - 137, - 138, - 139, - 140, - 141, - 142, - 143, - 144, - 145, - 146, - 147, - 148, - 149, - 150, - 151, - 152, - 153, - 154, - 155, - 156, - 157, - 158, - 159, - 160, - 161, - 162, - 163, - 164, - 165, - 166, - 167, - 168, - 169, - 170, - 171, - 172, - 173, - 174, - 175, - 176, - 177, - 178, - 179, - 180, - 181, - 182, - 183, - 184, - 185, - 186, - 187, - 188, - 189, - 190, - 191, - 192, - 193, - 194, - 195, - 196, - 197, - 198, - 199, - 200 - ], - "xaxis": "x", - "y": [ - 107126.58, - 214213.66, - 322468.69, - 427406.37, - 535728.44, - 643227.5, - 746997.96, - 852324.44, - 898199.42, - 994921.42, - 1086491.72, - 1188114.32, - 1275638.92, - 1373504.32, - 1456611.99, - 1547050.37, - 1556304.25, - 1498337.65, - 1480610.39, - 1442181.26, - 1380390.08, - 1315149.42, - 1250840.52, - 1198962.91, - 1113901.96, - 1112866.02, - 1109572.68, - 1112386.81, - 1104839.85, - 1088259.95, - 1072951.12, - 1038335.3, - 1039005.59, - 1055683.11, - 1074708.24, - 1088882.64, - 1084369.02, - 1061476.09, - 1037330.81, - 1001283.07, - 1011982.42, - 1039061.07, - 1060191.68, - 1069379.92, - 1051676.06, - 1025702.93, - 1000795.88, - 959941, - 928333.9, - 936392.44, - 947163.68, - 958614.58, - 973982.54, - 976113.12, - 983432.87, - 985159.38, - 949664.42, - 960540.52, - 971717.37, - 978223.94, - 995090.76, - 1000123.55, - 1006856.18, - 1013280.29, - 977531.19, - 988260.54, - 996765.65, - 1006933, - 1016151.03, - 1020419.88, - 1024544.66, - 1027862.2, - 989415.52, - 999208.44, - 1009747.84, - 1016122.1, - 1026766.44, - 1032416.84, - 1037369.06, - 1037677.89, - 1001527.34, - 1008569.78, - 1024112.93, - 1033177.84, - 1035389.26, - 1040484.52, - 1047416.67, - 1043614.54, - 1014160.19, - 1019409.94, - 1033667.5, - 1040422.32, - 1045409.98, - 1048162.62, - 1050384.15, - 1050304.88, - 1037251.1, - 1023279.61, - 1009889.86, - 996157.16, - 973425.48, - 960922.5, - 941705.52, - 927206.03, - 919716.12, - 907116.8, - 898444.05, - 889678.68, - 871535.65, - 858369.28, - 839357.6, - 828077.49, - 817619.99, - 806563.34, - 795820.84, - 789602.32, - 769744.98, - 754704.16, - 739965.24, - 721357.74, - 705584.89, - 689179.3, - 674153.86, - 668264.05, - 648129.94, - 630733.08, - 614518.38, - 598284.67, - 580642.38, - 562735.32, - 547668.6, - 540727.65, - 519637, - 499189.04, - 482457.86, - 458655.34, - 453087.56, - 445650.76, - 438779.54, - 434421.26, - 422130, - 403403.62, - 391528.74, - 374715.06, - 372678.44, - 371466.33, - 369815.4, - 367734.06, - 366332.54, - 365256.89, - 362078.84, - 361083.46, - 359994.43, - 359950.89, - 357498.64, - 357285.14, - 355405.08, - 354127.44, - 351793.59, - 350348, - 349438.44, - 349188.38, - 348377.38, - 347124.06, - 346480.82, - 345660.61, - 344352.86, - 343903.25, - 342402.74, - 342935.7, - 342089.26, - 341369.47, - 340166.19, - 339692.96, - 339204.8, - 338925.12, - 337700.46, - 338203.76, - 337556.9, - 336873.92, - 336399.84, - 336455.79, - 335823.56, - 335587.52, - 335620.09, - 334996.68, - 334980.98, - 335404.84, - 335051.8, - 334887.42, - 335150.96, - 334773.71, - 335035.28, - 334596.91, - 336065.8, - 335034.33, - 334931.36, - 334191.98, - 332929.11, - 331957.22 - ], - "yaxis": "y" - } - ], - "layout": { - "height": 400, - "legend": { - "title": { - "text": "Configuration" - }, - "tracegroupgap": 0 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "heatmapgl": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmapgl" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "Fuzzer performance" - }, - "width": 1200, - "xaxis": { - "anchor": "y", - "domain": [ - 0, - 1 - ], - "tickvals": [ - 0, - 5, - 10, - 15, - 20, - 25, - 30, - 35, - 40, - 45, - 50, - 55, - 60, - 65, - 70, - 75, - 80, - 85, - 90, - 95, - 100, - 105, - 110, - 115, - 120, - 125, - 130, - 135, - 140, - 145, - 150, - 155, - 160, - 165, - 170, - 175, - 180, - 185, - 190, - 195, - 200 - ], - "title": { - "text": "Number of parallel fuzzers" - } - }, - "yaxis": { - "anchor": "x", - "domain": [ - 0, - 1 - ], - "ticktext": [ - "10x", - "36x", - "62x", - "89x", - "115x", - "141x" - ], - "tickvals": [ - 105839.1, - 395932.13, - 686025.1599999999, - 976118.1899999998, - 1266211.22, - 1556304.25 - ], - "title": { - "text": "Fuzz target executions per second" - } - } - } - }, - "text/html": [ - "
" + "image/svg+xml": [ + "510152025303540455055606570758085909510010511011512012513013514014515015516016517017518018519019520010x36x62x89x115x141xConfigurationMulticore: Persistent mode/shared memory + kernel configMulticore: afl_execs: Persistent mode/shared memory + kernel configFuzzer performanceNumber of parallel fuzzersFuzz target executions per second" ] }, "metadata": {}, @@ -4436,7 +1071,8 @@ "# Update the primary Y-axis with custom tick labels\n", "r6a_fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n", "r6a_fig.update_xaxes(tickvals=list(range(0,200+1, 5)))\n", - "r6a_fig.update_layout(width=1200, height=400)" + "r6a_fig.update_layout(width=1200, height=400)\n", + "r6a_fig.show(\"svg\")" ] }, { @@ -4449,19 +1085,7 @@ "\n", "Does this mean that AFL++ is a bad fuzzer, or that AWS tricked us and gave us a 16-thread machine instead of a 192-thread one?\n", "\n", - "No -- the most likely cause here (based on a tip from @eqv) is that we're actually saturating the Linux kernel's ability to service system calls. We could look to reduce these, but there's another option available to us, which is to try running more system-call-servicers (read: kernels) at once, on this machine. One way to do that is to use hardware virtualization with KVM,\n", - "and if it is true that this particular fuzzer setup bottlenecks around 16 fuzzers, then we might expect an optimal number of KVM\n", - "kernels running on this machine to be around 16/192 == 8, each with 16 fuzzers in parallel, and perhaps a shared (in-memory?)\n", - "filesystem for the fuzzing queue." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Measuring system call saturation and experimenting with extra KVM hosts\n", - "\n", - "Coming soon!" + "No, probably not -- the most likely causes here are a problem with our Python harness, or potentially that we're already saturating the Linux kernel's ability to service system calls, although we're definitely hitting such a limit way earlier than expected. A good way to test this theory would be to run more system-call-servicers (read: kernels!) at once on this machine; one way to do that is to use hardware virtualization with KVM. " ] } ], diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index da32167a..e0dea299 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -64,7 +64,7 @@ cpu_count = multiprocessing.cpu_count() parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-b", "--basedir", help="directory to use for temp files", type=str, default="/tmp/aflpp-benchmark") parser.add_argument("-d", "--debug", help="show verbose debugging output", action="store_true") -parser.add_argument("-r", "--runs", help="how many runs to average results over", type=int, default=5) +parser.add_argument("-r", "--runs", help="how many runs to average results over", type=int, default=3) parser.add_argument("-f", "--fuzzers", help="how many afl-fuzz workers to use", type=int, default=cpu_count) parser.add_argument("-m", "--mode", help="pick modes", action="append", default=["multicore"], choices=mode_names) parser.add_argument("-c", "--comment", help="add a comment about your setup", type=str, default="") -- cgit 1.4.1