From b9db6b1254c9bf3a47c171bb96468628e9bd00f2 Mon Sep 17 00:00:00 2001 From: Chris Ball Date: Mon, 2 Oct 2023 03:23:09 -0700 Subject: benchmark: add a README, lower default runs from 5 to 3 --- benchmark/benchmark.ipynb | 3502 +-------------------------------------------- 1 file changed, 63 insertions(+), 3439 deletions(-) (limited to 'benchmark/benchmark.ipynb') diff --git a/benchmark/benchmark.ipynb b/benchmark/benchmark.ipynb index b1ef6b0f..7ff90fcd 100644 --- a/benchmark/benchmark.ipynb +++ b/benchmark/benchmark.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 502, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -20,14 +20,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Translate the JSON Lines entries into a single pandas DataFrame.\n", + "### Translate the JSON Lines entries into a single pandas DataFrame\n", "\n", "We have JSON Lines in [benchmark-results.jsonl](benchmark-results.jsonl) that look like this:" ] }, { "cell_type": "code", - "execution_count": 503, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 504, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -371,7 +371,7 @@ "[5 rows x 37 columns]" ] }, - "execution_count": 504, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -404,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 505, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -418,1647 +418,65 @@ }, { "cell_type": "code", - "execution_count": 506, + "execution_count": 16, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
execs_per_secparallel_fuzzersafl_persistent_configafl_system_configlabel
108135613.261.0TrueTrueSinglecore: Persistent mode/shared memory + ke...
117135613.2610.0TrueTrueSinglecore: Persistent mode/shared memory + ke...
\n", - "
" - ], - "text/plain": [ - " execs_per_sec parallel_fuzzers afl_persistent_config \\\n", - "108 135613.26 1.0 True \n", - "117 135613.26 10.0 True \n", - "\n", - " afl_system_config label \n", - "108 True Singlecore: Persistent mode/shared memory + ke... \n", - "117 True Singlecore: Persistent mode/shared memory + ke... " - ] - }, - "execution_count": 506, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "def build_graphdf_from_query(query: pd.DataFrame):\n", " \"\"\"Build a table suitable for graphing from a subset of the dataframe.\"\"\"\n", " graphdata = []\n", " max_fuzzers = int(query[[\"targets.test-instr-persist-shmem.multicore.fuzzers_used\", \"targets.test-instr.multicore.fuzzers_used\"]].max(axis=1).max(axis=0))\n", " for _, row in query.iterrows():\n", - " if row[\"targets.test-instr-persist-shmem.multicore.total_execs_per_sec\"] > 0:\n", - " execs_per_sec = row[\"targets.test-instr-persist-shmem.multicore.total_execs_per_sec\"]\n", - " afl_execs_per_sec = row[\"targets.test-instr-persist-shmem.multicore.afl_execs_per_sec\"]\n", - " parallel_fuzzers = row[\"targets.test-instr-persist-shmem.multicore.fuzzers_used\"]\n", - " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", - " afl_system_config = row[\"config.afl_system_config\"]\n", - " label = \"shmem-multicore\"\n", - " if afl_persistent_config:\n", - " label += \"+persist-conf\"\n", - " if afl_system_config:\n", - " label += \"+system-conf\"\n", - " if label == \"shmem-multicore+persist-conf+system-conf\":\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory + kernel config\"})\n", - " graphdata.append({\"execs_per_sec\": afl_execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: afl_execs: Persistent mode/shared memory + kernel config\"})\n", - " if label == \"shmem-multicore\":\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory without kernel config\"})\n", - " if row[\"targets.test-instr.multicore.total_execs_per_sec\"] > 0:\n", - " execs_per_sec = row[\"targets.test-instr.multicore.total_execs_per_sec\"]\n", - " parallel_fuzzers = row[\"targets.test-instr.multicore.fuzzers_used\"]\n", - " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", - " afl_system_config = row[\"config.afl_system_config\"]\n", - " label = \"base-multicore\"\n", - " if afl_persistent_config:\n", - " label += \"+persist-conf\"\n", - " if afl_system_config:\n", - " label += \"+system-conf\"\n", + " for target in [\"test-instr-persist-shmem\", \"test-instr\"]:\n", + " for mode in [\"multicore\", \"singlecore\"]:\n", + " label = \"\"\n", + " if not row[f\"targets.{target}.{mode}.total_execs_per_sec\"] > 0:\n", + " continue\n", + " execs_per_sec = row[f\"targets.{target}.{mode}.total_execs_per_sec\"]\n", + " afl_execs_per_sec = row[f\"targets.{target}.{mode}.afl_execs_per_sec\"]\n", + " parallel_fuzzers = row[f\"targets.{target}.{mode}.fuzzers_used\"]\n", + " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", + " afl_system_config = row[\"config.afl_system_config\"]\n", + " if target == \"test-instr-persist-shmem\":\n", + " label += \"shmem\"\n", + " else:\n", + " label += \"base\"\n", + " if mode == \"multicore\":\n", + " label += \"-multicore\"\n", + " else:\n", + " label += \"-singlecore\"\n", + " if afl_persistent_config:\n", + " label += \"+persist-conf\"\n", + " if afl_system_config:\n", + " label += \"+system-conf\"\n", + " \n", + " if label == \"shmem-multicore+persist-conf+system-conf\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory + kernel config\"})\n", + " graphdata.append({\"execs_per_sec\": afl_execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: afl_execs: Persistent mode/shared memory + kernel config\"})\n", + " if label == \"shmem-multicore\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory without kernel config\"})\n", " if label == \"base-multicore+persist-conf+system-conf\":\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Non-persistent mode + kernel config\"}) \n", - " if row[\"targets.test-instr-persist-shmem.singlecore.total_execs_per_sec\"] > 0:\n", - " execs_per_sec = row[\"targets.test-instr-persist-shmem.singlecore.total_execs_per_sec\"]\n", - " parallel_fuzzers = row[\"targets.test-instr-persist-shmem.singlecore.fuzzers_used\"]\n", - " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", - " afl_system_config = row[\"config.afl_system_config\"]\n", - " label = \"shmem-singlecore\"\n", - " if afl_persistent_config:\n", - " label += \"+persist-conf\"\n", - " if afl_system_config:\n", - " label += \"+system-conf\"\n", - " if label == \"shmem-singlecore+persist-conf+system-conf\":\n", - " for i in range(1, max_fuzzers + 1):\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Persistent mode/shared memory + kernel config\"})\n", - " if row[\"targets.test-instr.singlecore.total_execs_per_sec\"] > 0:\n", - " execs_per_sec = row[\"targets.test-instr.singlecore.total_execs_per_sec\"]\n", - " parallel_fuzzers = row[\"targets.test-instr.singlecore.fuzzers_used\"]\n", - " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", - " afl_system_config = row[\"config.afl_system_config\"]\n", - " label = \"base-singlecore\"\n", - " if afl_persistent_config:\n", - " label += \"+persist-conf\"\n", - " if afl_system_config:\n", - " label += \"+system-conf\"\n", - " if label == \"base-singlecore+persist-conf+system-conf\":\n", - " for i in range(1, max_fuzzers + 1):\n", - " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Non-persistent mode + kernel config\"})\n", - "\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Non-persistent mode + kernel config\"})\n", + " if label == \"shmem-singlecore+persist-conf+system-conf\":\n", + " for i in range(1, max_fuzzers + 1):\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Persistent mode/shared memory + kernel config\"})\n", + " if label == \"base-singlecore+persist-conf+system-conf\":\n", + " for i in range(1, max_fuzzers + 1):\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Non-persistent mode + kernel config\"})\n", " return pd.DataFrame.from_records(graphdata).sort_values(\"label\", ascending=False)\n", "\n", - "graphdf = build_graphdf_from_query(i7)\n", - "graphdf.head(2)" + "graphdf = build_graphdf_from_query(i7)" ] }, { "cell_type": "code", - "execution_count": 507, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "hovertemplate": "Configuration=Multicore: Non-persistent mode + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: Non-persistent mode + kernel config", - "line": { - "color": "#636efa", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: Non-persistent mode + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 11019.3, - 21111.92, - 30568.82, - 38963.07, - 47693.65, - 55718.73, - 64156.79, - 72176.39, - 61257.76, - 67507.14, - 73183.59, - 79167.7, - 85202.55, - 91594.86, - 97682.33, - 103765.38, - 84547.71, - 86611.67, - 88657, - 90134.42, - 91033.28, - 91637.86, - 92747.81, - 93207.38, - 92970.87, - 94162.8, - 94237.96, - 91716.1, - 94072.6, - 95644.79, - 94880.56, - 93460.57, - 94194.83, - 93853.08, - 95147.78, - 92697.06 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: Persistent mode/shared memory + kernel config", - "line": { - "color": "#EF553B", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 133591.26, - 255995.07, - 380246.34, - 490254.72, - 598698.16, - 716786.21, - 824873.02, - 942712.02, - 709716.24, - 779115.44, - 851918.03, - 914375.37, - 990573.31, - 1060551.02, - 1134650.66, - 1203287.99, - 1022498.84, - 1058151.58, - 1076742.64, - 1091743.7, - 1062616.36, - 1081621.57, - 1132929.86, - 1133825.45, - 1114215.27, - 1122210.96, - 1130627.8, - 1135890.71, - 1137390.94, - 1142969.21, - 1149056.35, - 1142131.87, - 1128973.67, - 1116863.53, - 1117913.34, - 1124962.18 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory without kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: Persistent mode/shared memory without kernel config", - "line": { - "color": "#00cc96", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: Persistent mode/shared memory without kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 90851.4, - 176159.32, - 260268.78, - 336355.99, - 413750, - 492578.2, - 565737.17, - 640579.35, - 491284.66, - 540759.63, - 591144.78, - 638938.52, - 687954.18, - 734886.87, - 784210.26, - 834811.24, - 714178.66, - 734804.4, - 740714.93, - 750425.99, - 752625.52, - 760661.34, - 758401.65, - 767985.22, - 785474.61, - 789679.78, - 790483.94, - 798176.69, - 792763.39, - 788972.72, - 789307.69, - 777440.02, - 767985.22, - 775967.55, - 764863.41, - 768616.31 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Multicore: afl_execs: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", - "line": { - "color": "#ab63fa", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 134423.5, - 258490.04, - 383777.45, - 496249.48, - 613089.31, - 730366.19, - 844187.32, - 962846.18, - 997414.74, - 1034757.73, - 1070703.42, - 1104249.08, - 1131176.88, - 1164076.48, - 1198824.47, - 1227578.7, - 1272311.96, - 1295688.8, - 1314398.6, - 1328581.94, - 1342660.66, - 1363930.3, - 1377043.72, - 1375818.24, - 1361687.56, - 1369637.56, - 1375444.16, - 1349599.77, - 1321658.08, - 1301868.24, - 1276904.9, - 1243444.8, - 1243981.21, - 1234425.98, - 1244349.38, - 1250454.58 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Singlecore: Non-persistent mode + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Singlecore: Non-persistent mode + kernel config", - "line": { - "color": "#FFA15A", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Singlecore: Non-persistent mode + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96, - 11038.96 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Singlecore: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Singlecore: Persistent mode/shared memory + kernel config", - "line": { - "color": "#19d3f3", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Singlecore: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "xaxis": "x", - "y": [ - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26, - 135613.26 - ], - "yaxis": "y" - } - ], - "layout": { - "height": 400, - "legend": { - "title": { - "text": "Configuration" - }, - "tracegroupgap": 0 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "heatmapgl": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmapgl" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "Fuzzer performance" - }, - "width": 1200, - "xaxis": { - "anchor": "y", - "domain": [ - 0, - 1 - ], - "tickvals": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36 - ], - "title": { - "text": "Number of parallel fuzzers" - } - }, - "yaxis": { - "anchor": "x", - "domain": [ - 0, - 1 - ], - "ticktext": [ - "1x", - "26x", - "51x", - "75x", - "100x", - "125x" - ], - "tickvals": [ - 11019.3, - 284224.18399999995, - 557429.068, - 830633.9519999999, - 1103838.836, - 1377043.72 - ], - "title": { - "text": "Fuzz target executions per second" - } - } - } - }, - "text/html": [ - "
" + "image/svg+xml": [ + "1234567891011121314151617181920212223242526272829303132333435361x26x51x75x100x125xConfigurationMulticore: Non-persistent mode + kernel configMulticore: Persistent mode/shared memory + kernel configMulticore: Persistent mode/shared memory without kernel configMulticore: afl_execs: Persistent mode/shared memory + kernel configSinglecore: Non-persistent mode + kernel configSinglecore: Persistent mode/shared memory + kernel configFuzzer performanceNumber of parallel fuzzersFuzz target executions per second" ] }, "metadata": {}, @@ -2066,7 +484,6 @@ } ], "source": [ - "\n", "import numpy as np\n", "pd.options.plotting.backend = \"plotly\"\n", "\n", @@ -2088,7 +505,8 @@ "# Update the primary Y-axis with custom tick labels\n", "fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n", "fig.update_xaxes(tickvals=list(range(1,36+1)))\n", - "fig.update_layout(width=1200, height=400)\n" + "fig.update_layout(width=1200, height=400)\n", + "fig.show(\"svg\")\n" ] }, { @@ -2100,7 +518,7 @@ }, { "cell_type": "code", - "execution_count": 508, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -2241,7 +659,7 @@ "5.0 135613.26 " ] }, - "execution_count": 508, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -2259,7 +677,7 @@ }, { "cell_type": "code", - "execution_count": 509, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -2318,7 +736,7 @@ "" ] }, - "execution_count": 509, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -2402,7 +820,7 @@ }, { "cell_type": "code", - "execution_count": 521, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -2537,7 +955,7 @@ "149 6.15 " ] }, - "execution_count": 521, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -2549,7 +967,7 @@ }, { "cell_type": "code", - "execution_count": 516, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -2611,7 +1029,7 @@ "153 True Multicore: afl_execs: Persistent mode/shared m... " ] }, - "execution_count": 516, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -2623,1796 +1041,13 @@ }, { "cell_type": "code", - "execution_count": 514, + "execution_count": 22, "metadata": {}, "outputs": [ { "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: Persistent mode/shared memory + kernel config", - "line": { - "color": "#636efa", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 58, - 59, - 60, - 61, - 62, - 63, - 64, - 65, - 66, - 67, - 68, - 69, - 70, - 71, - 72, - 73, - 74, - 75, - 76, - 77, - 78, - 79, - 80, - 81, - 82, - 83, - 84, - 85, - 86, - 87, - 88, - 89, - 90, - 91, - 92, - 93, - 94, - 95, - 96, - 97, - 98, - 99, - 100, - 101, - 102, - 103, - 104, - 105, - 106, - 107, - 108, - 109, - 110, - 111, - 112, - 113, - 114, - 115, - 116, - 117, - 118, - 119, - 120, - 121, - 122, - 123, - 124, - 125, - 126, - 127, - 128, - 129, - 130, - 131, - 132, - 133, - 134, - 135, - 136, - 137, - 138, - 139, - 140, - 141, - 142, - 143, - 144, - 145, - 146, - 147, - 148, - 149, - 150, - 151, - 152, - 153, - 154, - 155, - 156, - 157, - 158, - 159, - 160, - 161, - 162, - 163, - 164, - 165, - 166, - 167, - 168, - 169, - 170, - 171, - 172, - 173, - 174, - 175, - 176, - 177, - 178, - 179, - 180, - 181, - 182, - 183, - 184, - 185, - 186, - 187, - 188, - 189, - 190, - 191, - 192, - 193, - 194, - 195, - 196, - 197, - 198, - 199, - 200 - ], - "xaxis": "x", - "y": [ - 105839.1, - 210819.47, - 316229.21, - 413256.46, - 518632.73, - 619884.69, - 702256.76, - 804131.53, - 847288.04, - 943139.75, - 1024439.07, - 1113578.57, - 1162772.81, - 1265283.48, - 1346295.34, - 1421319.66, - 1389055.03, - 1336294.29, - 1354421.12, - 1341083.87, - 1267487.8, - 1228006.44, - 1164952.24, - 1117569.89, - 1027015.81, - 1011333.83, - 993703.26, - 983824.21, - 969783.14, - 953522.94, - 937704.89, - 914207.81, - 898800.31, - 896437.34, - 895982.76, - 896412.07, - 888119.63, - 874168.22, - 858049.53, - 839870.71, - 830337.88, - 833695.19, - 832246.18, - 831472, - 820819.59, - 809235.61, - 796104.63, - 779748.67, - 651915.77, - 658310.11, - 664906.42, - 670874.88, - 679223.43, - 680460.23, - 683449.31, - 683776.32, - 642820.96, - 656234.7, - 664079.06, - 668815.96, - 677202.95, - 682183.78, - 687798.53, - 695065.41, - 653862.76, - 662255.65, - 667137.25, - 674252.38, - 674769.27, - 676779.76, - 678994.74, - 677707.66, - 643636.07, - 645011.41, - 647860.09, - 650443.61, - 655762.21, - 655894.58, - 659395.36, - 660632.48, - 636619.43, - 636679.37, - 641091.3, - 642134.44, - 644188.04, - 645646.45, - 648190.91, - 645188.95, - 623576.69, - 623521.92, - 627188.81, - 631735.31, - 632169.99, - 632104.98, - 636441.28, - 636413.09, - 627747.11, - 618580.91, - 610289.87, - 602517.1, - 587297.53, - 574782.55, - 559428.41, - 543938.93, - 538120.81, - 531555.69, - 526211.57, - 521796.77, - 510905.75, - 501877.11, - 490630.76, - 479788.36, - 475372.8, - 470776.14, - 466163.29, - 463421.1, - 454522.49, - 446589.03, - 438649.5, - 428977.64, - 424292.46, - 418867.74, - 414980.74, - 412384.2, - 404224.17, - 396310.94, - 389392.67, - 381715.99, - 377825.09, - 374215.78, - 370708.15, - 368402.32, - 361940.65, - 355502.6, - 349589.01, - 342501.52, - 339897.15, - 337989.11, - 335723.22, - 334039.83, - 329127.04, - 323073.5, - 318652.21, - 313198.69, - 311591.71, - 310530.42, - 309390.36, - 307977.56, - 306559.97, - 305066.46, - 302157.29, - 301097.52, - 300138.11, - 299824.88, - 298130.63, - 297838.27, - 296189.21, - 294695.11, - 292518.21, - 291030.08, - 290207.87, - 289828.9, - 289247.99, - 288295.58, - 287447.58, - 286405.31, - 284755.57, - 284046.79, - 283176.27, - 283053.87, - 282178.36, - 281522.15, - 280377.23, - 279885.55, - 278793.08, - 277923.89, - 277069.78, - 277124.24, - 276576.49, - 276020.64, - 275328, - 275029.74, - 274030.79, - 273612.38, - 273285.13, - 273039.33, - 272819.57, - 272960.42, - 272388.01, - 272311.26, - 272115.97, - 272056.42, - 271835.4, - 271397.63, - 271867.2, - 271065.21, - 270797.96, - 270150.29, - 269442.01, - 268674.91 - ], - "yaxis": "y" - }, - { - "hovertemplate": "Configuration=Multicore: afl_execs: Persistent mode/shared memory + kernel config
Number of parallel fuzzers=%{x}
Fuzz target executions per second=%{y}", - "legendgroup": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", - "line": { - "color": "#EF553B", - "dash": "solid" - }, - "marker": { - "symbol": "circle" - }, - "mode": "lines", - "name": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", - "orientation": "v", - "showlegend": true, - "type": "scatter", - "x": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 58, - 59, - 60, - 61, - 62, - 63, - 64, - 65, - 66, - 67, - 68, - 69, - 70, - 71, - 72, - 73, - 74, - 75, - 76, - 77, - 78, - 79, - 80, - 81, - 82, - 83, - 84, - 85, - 86, - 87, - 88, - 89, - 90, - 91, - 92, - 93, - 94, - 95, - 96, - 97, - 98, - 99, - 100, - 101, - 102, - 103, - 104, - 105, - 106, - 107, - 108, - 109, - 110, - 111, - 112, - 113, - 114, - 115, - 116, - 117, - 118, - 119, - 120, - 121, - 122, - 123, - 124, - 125, - 126, - 127, - 128, - 129, - 130, - 131, - 132, - 133, - 134, - 135, - 136, - 137, - 138, - 139, - 140, - 141, - 142, - 143, - 144, - 145, - 146, - 147, - 148, - 149, - 150, - 151, - 152, - 153, - 154, - 155, - 156, - 157, - 158, - 159, - 160, - 161, - 162, - 163, - 164, - 165, - 166, - 167, - 168, - 169, - 170, - 171, - 172, - 173, - 174, - 175, - 176, - 177, - 178, - 179, - 180, - 181, - 182, - 183, - 184, - 185, - 186, - 187, - 188, - 189, - 190, - 191, - 192, - 193, - 194, - 195, - 196, - 197, - 198, - 199, - 200 - ], - "xaxis": "x", - "y": [ - 107126.58, - 214213.66, - 322468.69, - 427406.37, - 535728.44, - 643227.5, - 746997.96, - 852324.44, - 898199.42, - 994921.42, - 1086491.72, - 1188114.32, - 1275638.92, - 1373504.32, - 1456611.99, - 1547050.37, - 1556304.25, - 1498337.65, - 1480610.39, - 1442181.26, - 1380390.08, - 1315149.42, - 1250840.52, - 1198962.91, - 1113901.96, - 1112866.02, - 1109572.68, - 1112386.81, - 1104839.85, - 1088259.95, - 1072951.12, - 1038335.3, - 1039005.59, - 1055683.11, - 1074708.24, - 1088882.64, - 1084369.02, - 1061476.09, - 1037330.81, - 1001283.07, - 1011982.42, - 1039061.07, - 1060191.68, - 1069379.92, - 1051676.06, - 1025702.93, - 1000795.88, - 959941, - 928333.9, - 936392.44, - 947163.68, - 958614.58, - 973982.54, - 976113.12, - 983432.87, - 985159.38, - 949664.42, - 960540.52, - 971717.37, - 978223.94, - 995090.76, - 1000123.55, - 1006856.18, - 1013280.29, - 977531.19, - 988260.54, - 996765.65, - 1006933, - 1016151.03, - 1020419.88, - 1024544.66, - 1027862.2, - 989415.52, - 999208.44, - 1009747.84, - 1016122.1, - 1026766.44, - 1032416.84, - 1037369.06, - 1037677.89, - 1001527.34, - 1008569.78, - 1024112.93, - 1033177.84, - 1035389.26, - 1040484.52, - 1047416.67, - 1043614.54, - 1014160.19, - 1019409.94, - 1033667.5, - 1040422.32, - 1045409.98, - 1048162.62, - 1050384.15, - 1050304.88, - 1037251.1, - 1023279.61, - 1009889.86, - 996157.16, - 973425.48, - 960922.5, - 941705.52, - 927206.03, - 919716.12, - 907116.8, - 898444.05, - 889678.68, - 871535.65, - 858369.28, - 839357.6, - 828077.49, - 817619.99, - 806563.34, - 795820.84, - 789602.32, - 769744.98, - 754704.16, - 739965.24, - 721357.74, - 705584.89, - 689179.3, - 674153.86, - 668264.05, - 648129.94, - 630733.08, - 614518.38, - 598284.67, - 580642.38, - 562735.32, - 547668.6, - 540727.65, - 519637, - 499189.04, - 482457.86, - 458655.34, - 453087.56, - 445650.76, - 438779.54, - 434421.26, - 422130, - 403403.62, - 391528.74, - 374715.06, - 372678.44, - 371466.33, - 369815.4, - 367734.06, - 366332.54, - 365256.89, - 362078.84, - 361083.46, - 359994.43, - 359950.89, - 357498.64, - 357285.14, - 355405.08, - 354127.44, - 351793.59, - 350348, - 349438.44, - 349188.38, - 348377.38, - 347124.06, - 346480.82, - 345660.61, - 344352.86, - 343903.25, - 342402.74, - 342935.7, - 342089.26, - 341369.47, - 340166.19, - 339692.96, - 339204.8, - 338925.12, - 337700.46, - 338203.76, - 337556.9, - 336873.92, - 336399.84, - 336455.79, - 335823.56, - 335587.52, - 335620.09, - 334996.68, - 334980.98, - 335404.84, - 335051.8, - 334887.42, - 335150.96, - 334773.71, - 335035.28, - 334596.91, - 336065.8, - 335034.33, - 334931.36, - 334191.98, - 332929.11, - 331957.22 - ], - "yaxis": "y" - } - ], - "layout": { - "height": 400, - "legend": { - "title": { - "text": "Configuration" - }, - "tracegroupgap": 0 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "heatmapgl": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmapgl" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "Fuzzer performance" - }, - "width": 1200, - "xaxis": { - "anchor": "y", - "domain": [ - 0, - 1 - ], - "tickvals": [ - 0, - 5, - 10, - 15, - 20, - 25, - 30, - 35, - 40, - 45, - 50, - 55, - 60, - 65, - 70, - 75, - 80, - 85, - 90, - 95, - 100, - 105, - 110, - 115, - 120, - 125, - 130, - 135, - 140, - 145, - 150, - 155, - 160, - 165, - 170, - 175, - 180, - 185, - 190, - 195, - 200 - ], - "title": { - "text": "Number of parallel fuzzers" - } - }, - "yaxis": { - "anchor": "x", - "domain": [ - 0, - 1 - ], - "ticktext": [ - "10x", - "36x", - "62x", - "89x", - "115x", - "141x" - ], - "tickvals": [ - 105839.1, - 395932.13, - 686025.1599999999, - 976118.1899999998, - 1266211.22, - 1556304.25 - ], - "title": { - "text": "Fuzz target executions per second" - } - } - } - }, - "text/html": [ - "
" + "image/svg+xml": [ + "510152025303540455055606570758085909510010511011512012513013514014515015516016517017518018519019520010x36x62x89x115x141xConfigurationMulticore: Persistent mode/shared memory + kernel configMulticore: afl_execs: Persistent mode/shared memory + kernel configFuzzer performanceNumber of parallel fuzzersFuzz target executions per second" ] }, "metadata": {}, @@ -4436,7 +1071,8 @@ "# Update the primary Y-axis with custom tick labels\n", "r6a_fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n", "r6a_fig.update_xaxes(tickvals=list(range(0,200+1, 5)))\n", - "r6a_fig.update_layout(width=1200, height=400)" + "r6a_fig.update_layout(width=1200, height=400)\n", + "r6a_fig.show(\"svg\")" ] }, { @@ -4449,19 +1085,7 @@ "\n", "Does this mean that AFL++ is a bad fuzzer, or that AWS tricked us and gave us a 16-thread machine instead of a 192-thread one?\n", "\n", - "No -- the most likely cause here (based on a tip from @eqv) is that we're actually saturating the Linux kernel's ability to service system calls. We could look to reduce these, but there's another option available to us, which is to try running more system-call-servicers (read: kernels) at once, on this machine. One way to do that is to use hardware virtualization with KVM,\n", - "and if it is true that this particular fuzzer setup bottlenecks around 16 fuzzers, then we might expect an optimal number of KVM\n", - "kernels running on this machine to be around 16/192 == 8, each with 16 fuzzers in parallel, and perhaps a shared (in-memory?)\n", - "filesystem for the fuzzing queue." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Measuring system call saturation and experimenting with extra KVM hosts\n", - "\n", - "Coming soon!" + "No, probably not -- the most likely causes here are a problem with our Python harness, or potentially that we're already saturating the Linux kernel's ability to service system calls, although we're definitely hitting such a limit way earlier than expected. A good way to test this theory would be to run more system-call-servicers (read: kernels!) at once on this machine; one way to do that is to use hardware virtualization with KVM. " ] } ], -- cgit 1.4.1