diff options
Diffstat (limited to 'benchmark/benchmark.ipynb')
| -rw-r--r-- | benchmark/benchmark.ipynb | 4490 |
1 files changed, 4490 insertions, 0 deletions
diff --git a/benchmark/benchmark.ipynb b/benchmark/benchmark.ipynb new file mode 100644 index 00000000..b1ef6b0f --- /dev/null +++ b/benchmark/benchmark.ipynb @@ -0,0 +1,4490 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 502, + "metadata": {}, + "outputs": [], + "source": [ + "# benchmark.ipynb\n", + "# Part of the aflplusplus project, requires an ipynb (Jupyter) editor or viewer.\n", + "# Author: Chris Ball <chris@printf.net>\n", + "import json\n", + "import pandas as pd\n", + "with open(\"benchmark-results.jsonl\") as f:\n", + " lines = f.read().splitlines()\n", + "json_lines = [json.loads(line) for line in lines]\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Translate the JSON Lines entries into a single pandas DataFrame.\n", + "\n", + "We have JSON Lines in [benchmark-results.jsonl](benchmark-results.jsonl) that look like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 503, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"config\": {\n", + " \"afl_persistent_config\": true,\n", + " \"afl_system_config\": true,\n", + " \"afl_version\": \"++4.09a\",\n", + " \"comment\": \"i9-9900k, 16GB DDR4-3000, Arch Linux\",\n", + " \"compiler\": \"clang version 15.0.7\",\n", + " \"target_arch\": \"x86_64-pc-linux-gnu\"\n", + " },\n", + " \"hardware\": {\n", + " \"cpu_fastest_core_mhz\": 4999.879,\n", + " \"cpu_model\": \"Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz\",\n", + " \"cpu_threads\": 16\n", + " },\n", + " \"targets\": {\n", + " \"test-instr\": {\n", + " \"multicore\": {\n", + " \"afl_execs_per_sec\": 11025.88,\n", + " \"afl_execs_total\": 519670,\n", + " \"fuzzers_used\": 1,\n", + " \"run_end\": \"2023-09-24 01:18:19.516294\",\n", + " \"run_start\": \"2023-09-24 01:17:55.982600\",\n", + " \"total_execs_per_sec\": 11019.3,\n", + " \"total_run_time\": 47.16\n", + " }\n", + " },\n", + " \"test-instr-persist-shmem\": {\n", + " \"multicore\": {\n", + " \"afl_execs_per_sec\": 134423.5,\n", + " \"afl_execs_total\": 519670,\n", + " \"fuzzers_used\": 1,\n", + " \"run_end\": \"2023-09-24 01:17:32.262373\",\n", + " \"run_start\": \"2023-09-24 01:17:30.328037\",\n", + " \"total_execs_per_sec\": 133591.26,\n", + " \"total_run_time\": 3.89\n", + " }\n", + " }\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(json.loads(lines[0]), indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The [pd.json_normalize()](https://pandas.pydata.org/docs/reference/api/pandas.json_normalize.html]) method translates this into a flat table that we can perform queries against:" + ] + }, + { + "cell_type": "code", + "execution_count": 504, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>config.afl_persistent_config</th>\n", + " <th>config.afl_system_config</th>\n", + " <th>config.afl_version</th>\n", + " <th>config.comment</th>\n", + " <th>config.compiler</th>\n", + " <th>config.target_arch</th>\n", + " <th>hardware.cpu_fastest_core_mhz</th>\n", + " <th>hardware.cpu_model</th>\n", + " <th>hardware.cpu_threads</th>\n", + " <th>targets.test-instr.multicore.afl_execs_per_sec</th>\n", + " <th>...</th>\n", + " <th>targets.test-instr.singlecore.run_start</th>\n", + " <th>targets.test-instr.singlecore.total_execs_per_sec</th>\n", + " <th>targets.test-instr.singlecore.total_run_time</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.afl_execs_per_sec</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.afl_execs_total</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.fuzzers_used</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.run_end</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.run_start</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.total_execs_per_sec</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.total_run_time</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 15.0.7</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4999.879</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>11025.88</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 15.0.7</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4998.794</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>21139.64</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 15.0.7</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4998.859</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>30618.28</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 15.0.7</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>5000.078</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>39125.92</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 15.0.7</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4996.885</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>47861.04</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows × 37 columns</p>\n", + "</div>" + ], + "text/plain": [ + " config.afl_persistent_config config.afl_system_config config.afl_version \\\n", + "0 True True ++4.09a \n", + "1 True True ++4.09a \n", + "2 True True ++4.09a \n", + "3 True True ++4.09a \n", + "4 True True ++4.09a \n", + "\n", + " config.comment config.compiler \\\n", + "0 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 15.0.7 \n", + "1 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 15.0.7 \n", + "2 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 15.0.7 \n", + "3 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 15.0.7 \n", + "4 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 15.0.7 \n", + "\n", + " config.target_arch hardware.cpu_fastest_core_mhz \\\n", + "0 x86_64-pc-linux-gnu 4999.879 \n", + "1 x86_64-pc-linux-gnu 4998.794 \n", + "2 x86_64-pc-linux-gnu 4998.859 \n", + "3 x86_64-pc-linux-gnu 5000.078 \n", + "4 x86_64-pc-linux-gnu 4996.885 \n", + "\n", + " hardware.cpu_model hardware.cpu_threads \\\n", + "0 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "1 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "2 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "3 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "4 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "\n", + " targets.test-instr.multicore.afl_execs_per_sec ... \\\n", + "0 11025.88 ... \n", + "1 21139.64 ... \n", + "2 30618.28 ... \n", + "3 39125.92 ... \n", + "4 47861.04 ... \n", + "\n", + " targets.test-instr.singlecore.run_start \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr.singlecore.total_execs_per_sec \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr.singlecore.total_run_time \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.afl_execs_per_sec \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.afl_execs_total \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.fuzzers_used \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.run_end \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.run_start \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.total_execs_per_sec \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.total_run_time \n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + "[5 rows x 37 columns]" + ] + }, + "execution_count": 504, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "df = pd.json_normalize(json_lines)\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Graph prep\n", + "\n", + "We're looking for a line graph showing lines for the following cases:\n", + "\n", + "* For each mode:\n", + " * For each target:\n", + " * persistent off, system off\n", + " * persistent on, system off\n", + " * persistent off, system on\n", + " * persistent on, system on\n", + "\n", + "where the x-axis is number of cores, and the y-axis is either afl_execs_per_sec or total_execs_per_sec (I'm not yet sure which is a better metric to use).\n", + "\n", + "But first, a mini test harness by checking that the number of rows matched what we'd intuitively expect:" + ] + }, + { + "cell_type": "code", + "execution_count": 505, + "metadata": {}, + "outputs": [], + "source": [ + "i7 = df.query(\"`config.comment` == 'i9-9900k, 16GB DDR4-3000, Arch Linux'\")\n", + "assert len(i7) == 148\n", + "multicore = i7.query(\"`targets.test-instr-persist-shmem.multicore.total_execs_per_sec` > 0 or `targets.test-instr.multicore.total_execs_per_sec` > 0\")\n", + "assert len(multicore) == 144 # 36 cores * 4 states * 1 run (containing two targets)\n", + "singlecore = i7.query(\"`targets.test-instr-persist-shmem.singlecore.total_execs_per_sec` > 0 or `targets.test-instr.singlecore.total_execs_per_sec` > 0\")\n", + "assert len(singlecore) == 4 # 1 core * 4 states * 1 run (containing two targets)" + ] + }, + { + "cell_type": "code", + "execution_count": 506, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>execs_per_sec</th>\n", + " <th>parallel_fuzzers</th>\n", + " <th>afl_persistent_config</th>\n", + " <th>afl_system_config</th>\n", + " <th>label</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>108</th>\n", + " <td>135613.26</td>\n", + " <td>1.0</td>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>Singlecore: Persistent mode/shared memory + ke...</td>\n", + " </tr>\n", + " <tr>\n", + " <th>117</th>\n", + " <td>135613.26</td>\n", + " <td>10.0</td>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>Singlecore: Persistent mode/shared memory + ke...</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " execs_per_sec parallel_fuzzers afl_persistent_config \\\n", + "108 135613.26 1.0 True \n", + "117 135613.26 10.0 True \n", + "\n", + " afl_system_config label \n", + "108 True Singlecore: Persistent mode/shared memory + ke... \n", + "117 True Singlecore: Persistent mode/shared memory + ke... " + ] + }, + "execution_count": 506, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def build_graphdf_from_query(query: pd.DataFrame):\n", + " \"\"\"Build a table suitable for graphing from a subset of the dataframe.\"\"\"\n", + " graphdata = []\n", + " max_fuzzers = int(query[[\"targets.test-instr-persist-shmem.multicore.fuzzers_used\", \"targets.test-instr.multicore.fuzzers_used\"]].max(axis=1).max(axis=0))\n", + " for _, row in query.iterrows():\n", + " if row[\"targets.test-instr-persist-shmem.multicore.total_execs_per_sec\"] > 0:\n", + " execs_per_sec = row[\"targets.test-instr-persist-shmem.multicore.total_execs_per_sec\"]\n", + " afl_execs_per_sec = row[\"targets.test-instr-persist-shmem.multicore.afl_execs_per_sec\"]\n", + " parallel_fuzzers = row[\"targets.test-instr-persist-shmem.multicore.fuzzers_used\"]\n", + " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", + " afl_system_config = row[\"config.afl_system_config\"]\n", + " label = \"shmem-multicore\"\n", + " if afl_persistent_config:\n", + " label += \"+persist-conf\"\n", + " if afl_system_config:\n", + " label += \"+system-conf\"\n", + " if label == \"shmem-multicore+persist-conf+system-conf\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory + kernel config\"})\n", + " graphdata.append({\"execs_per_sec\": afl_execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: afl_execs: Persistent mode/shared memory + kernel config\"})\n", + " if label == \"shmem-multicore\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory without kernel config\"})\n", + " if row[\"targets.test-instr.multicore.total_execs_per_sec\"] > 0:\n", + " execs_per_sec = row[\"targets.test-instr.multicore.total_execs_per_sec\"]\n", + " parallel_fuzzers = row[\"targets.test-instr.multicore.fuzzers_used\"]\n", + " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", + " afl_system_config = row[\"config.afl_system_config\"]\n", + " label = \"base-multicore\"\n", + " if afl_persistent_config:\n", + " label += \"+persist-conf\"\n", + " if afl_system_config:\n", + " label += \"+system-conf\"\n", + " if label == \"base-multicore+persist-conf+system-conf\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Non-persistent mode + kernel config\"}) \n", + " if row[\"targets.test-instr-persist-shmem.singlecore.total_execs_per_sec\"] > 0:\n", + " execs_per_sec = row[\"targets.test-instr-persist-shmem.singlecore.total_execs_per_sec\"]\n", + " parallel_fuzzers = row[\"targets.test-instr-persist-shmem.singlecore.fuzzers_used\"]\n", + " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", + " afl_system_config = row[\"config.afl_system_config\"]\n", + " label = \"shmem-singlecore\"\n", + " if afl_persistent_config:\n", + " label += \"+persist-conf\"\n", + " if afl_system_config:\n", + " label += \"+system-conf\"\n", + " if label == \"shmem-singlecore+persist-conf+system-conf\":\n", + " for i in range(1, max_fuzzers + 1):\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Persistent mode/shared memory + kernel config\"})\n", + " if row[\"targets.test-instr.singlecore.total_execs_per_sec\"] > 0:\n", + " execs_per_sec = row[\"targets.test-instr.singlecore.total_execs_per_sec\"]\n", + " parallel_fuzzers = row[\"targets.test-instr.singlecore.fuzzers_used\"]\n", + " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", + " afl_system_config = row[\"config.afl_system_config\"]\n", + " label = \"base-singlecore\"\n", + " if afl_persistent_config:\n", + " label += \"+persist-conf\"\n", + " if afl_system_config:\n", + " label += \"+system-conf\"\n", + " if label == \"base-singlecore+persist-conf+system-conf\":\n", + " for i in range(1, max_fuzzers + 1):\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Non-persistent mode + kernel config\"})\n", + "\n", + " return pd.DataFrame.from_records(graphdata).sort_values(\"label\", ascending=False)\n", + "\n", + "graphdf = build_graphdf_from_query(i7)\n", + "graphdf.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 507, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "hovertemplate": "Configuration=Multicore: Non-persistent mode + kernel config<br>Number of parallel fuzzers=%{x}<br>Fuzz target executions per second=%{y}<extra></extra>", + "legendgroup": "Multicore: Non-persistent mode + kernel config", + "line": { + "color": "#636efa", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "Multicore: Non-persistent mode + kernel config", + "orientation": "v", + "showlegend": true, + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36 + ], + "xaxis": "x", + "y": [ + 11019.3, + 21111.92, + 30568.82, + 38963.07, + 47693.65, + 55718.73, + 64156.79, + 72176.39, + 61257.76, + 67507.14, + 73183.59, + 79167.7, + 85202.55, + 91594.86, + 97682.33, + 103765.38, + 84547.71, + 86611.67, + 88657, + 90134.42, + 91033.28, + 91637.86, + 92747.81, + 93207.38, + 92970.87, + 94162.8, + 94237.96, + 91716.1, + 94072.6, + 95644.79, + 94880.56, + 93460.57, + 94194.83, + 93853.08, + 95147.78, + 92697.06 + ], + "yaxis": "y" + }, + { + "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory + kernel config<br>Number of parallel fuzzers=%{x}<br>Fuzz target executions per second=%{y}<extra></extra>", + "legendgroup": "Multicore: Persistent mode/shared memory + kernel config", + "line": { + "color": "#EF553B", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "Multicore: Persistent mode/shared memory + kernel config", + "orientation": "v", + "showlegend": true, + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36 + ], + "xaxis": "x", + "y": [ + 133591.26, + 255995.07, + 380246.34, + 490254.72, + 598698.16, + 716786.21, + 824873.02, + 942712.02, + 709716.24, + 779115.44, + 851918.03, + 914375.37, + 990573.31, + 1060551.02, + 1134650.66, + 1203287.99, + 1022498.84, + 1058151.58, + 1076742.64, + 1091743.7, + 1062616.36, + 1081621.57, + 1132929.86, + 1133825.45, + 1114215.27, + 1122210.96, + 1130627.8, + 1135890.71, + 1137390.94, + 1142969.21, + 1149056.35, + 1142131.87, + 1128973.67, + 1116863.53, + 1117913.34, + 1124962.18 + ], + "yaxis": "y" + }, + { + "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory without kernel config<br>Number of parallel fuzzers=%{x}<br>Fuzz target executions per second=%{y}<extra></extra>", + "legendgroup": "Multicore: Persistent mode/shared memory without kernel config", + "line": { + "color": "#00cc96", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "Multicore: Persistent mode/shared memory without kernel config", + "orientation": "v", + "showlegend": true, + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36 + ], + "xaxis": "x", + "y": [ + 90851.4, + 176159.32, + 260268.78, + 336355.99, + 413750, + 492578.2, + 565737.17, + 640579.35, + 491284.66, + 540759.63, + 591144.78, + 638938.52, + 687954.18, + 734886.87, + 784210.26, + 834811.24, + 714178.66, + 734804.4, + 740714.93, + 750425.99, + 752625.52, + 760661.34, + 758401.65, + 767985.22, + 785474.61, + 789679.78, + 790483.94, + 798176.69, + 792763.39, + 788972.72, + 789307.69, + 777440.02, + 767985.22, + 775967.55, + 764863.41, + 768616.31 + ], + "yaxis": "y" + }, + { + "hovertemplate": "Configuration=Multicore: afl_execs: Persistent mode/shared memory + kernel config<br>Number of parallel fuzzers=%{x}<br>Fuzz target executions per second=%{y}<extra></extra>", + "legendgroup": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", + "line": { + "color": "#ab63fa", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", + "orientation": "v", + "showlegend": true, + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36 + ], + "xaxis": "x", + "y": [ + 134423.5, + 258490.04, + 383777.45, + 496249.48, + 613089.31, + 730366.19, + 844187.32, + 962846.18, + 997414.74, + 1034757.73, + 1070703.42, + 1104249.08, + 1131176.88, + 1164076.48, + 1198824.47, + 1227578.7, + 1272311.96, + 1295688.8, + 1314398.6, + 1328581.94, + 1342660.66, + 1363930.3, + 1377043.72, + 1375818.24, + 1361687.56, + 1369637.56, + 1375444.16, + 1349599.77, + 1321658.08, + 1301868.24, + 1276904.9, + 1243444.8, + 1243981.21, + 1234425.98, + 1244349.38, + 1250454.58 + ], + "yaxis": "y" + }, + { + "hovertemplate": "Configuration=Singlecore: Non-persistent mode + kernel config<br>Number of parallel fuzzers=%{x}<br>Fuzz target executions per second=%{y}<extra></extra>", + "legendgroup": "Singlecore: Non-persistent mode + kernel config", + "line": { + "color": "#FFA15A", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "Singlecore: Non-persistent mode + kernel config", + "orientation": "v", + "showlegend": true, + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36 + ], + "xaxis": "x", + "y": [ + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96, + 11038.96 + ], + "yaxis": "y" + }, + { + "hovertemplate": "Configuration=Singlecore: Persistent mode/shared memory + kernel config<br>Number of parallel fuzzers=%{x}<br>Fuzz target executions per second=%{y}<extra></extra>", + "legendgroup": "Singlecore: Persistent mode/shared memory + kernel config", + "line": { + "color": "#19d3f3", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "Singlecore: Persistent mode/shared memory + kernel config", + "orientation": "v", + "showlegend": true, + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36 + ], + "xaxis": "x", + "y": [ + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26, + 135613.26 + ], + "yaxis": "y" + } + ], + "layout": { + "height": 400, + "legend": { + "title": { + "text": "Configuration" + }, + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Fuzzer performance" + }, + "width": 1200, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "tickvals": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36 + ], + "title": { + "text": "Number of parallel fuzzers" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "ticktext": [ + "1x", + "26x", + "51x", + "75x", + "100x", + "125x" + ], + "tickvals": [ + 11019.3, + 284224.18399999995, + 557429.068, + 830633.9519999999, + 1103838.836, + 1377043.72 + ], + "title": { + "text": "Fuzz target executions per second" + } + } + } + }, + "text/html": [ + "<div> <div id=\"80d54fb8-8ae8-4543-ba3c-c1846c8bbc5b\" class=\"plotly-graph-div\" style=\"height:400px; width:1200px;\"></div> <script type=\"text/javascript\"> require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"80d54fb8-8ae8-4543-ba3c-c1846c8bbc5b\")) { Plotly.newPlot( \"80d54fb8-8ae8-4543-ba3c-c1846c8bbc5b\", [{\"hovertemplate\":\"Configuration=Multicore: Non-persistent mode + kernel config\\u003cbr\\u003eNumber of parallel fuzzers=%{x}\\u003cbr\\u003eFuzz target executions per second=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Multicore: Non-persistent mode + kernel config\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"Multicore: Non-persistent mode + kernel config\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0],\"xaxis\":\"x\",\"y\":[11019.3,21111.92,30568.82,38963.07,47693.65,55718.73,64156.79,72176.39,61257.76,67507.14,73183.59,79167.7,85202.55,91594.86,97682.33,103765.38,84547.71,86611.67,88657.0,90134.42,91033.28,91637.86,92747.81,93207.38,92970.87,94162.8,94237.96,91716.1,94072.6,95644.79,94880.56,93460.57,94194.83,93853.08,95147.78,92697.06],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"Configuration=Multicore: Persistent mode\\u002fshared memory + kernel config\\u003cbr\\u003eNumber of parallel fuzzers=%{x}\\u003cbr\\u003eFuzz target executions per second=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Multicore: Persistent mode\\u002fshared memory + kernel config\",\"line\":{\"color\":\"#EF553B\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"Multicore: Persistent mode\\u002fshared memory + kernel config\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0],\"xaxis\":\"x\",\"y\":[133591.26,255995.07,380246.34,490254.72,598698.16,716786.21,824873.02,942712.02,709716.24,779115.44,851918.03,914375.37,990573.31,1060551.02,1134650.66,1203287.99,1022498.84,1058151.58,1076742.64,1091743.7,1062616.36,1081621.57,1132929.86,1133825.45,1114215.27,1122210.96,1130627.8,1135890.71,1137390.94,1142969.21,1149056.35,1142131.87,1128973.67,1116863.53,1117913.34,1124962.18],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"Configuration=Multicore: Persistent mode\\u002fshared memory without kernel config\\u003cbr\\u003eNumber of parallel fuzzers=%{x}\\u003cbr\\u003eFuzz target executions per second=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Multicore: Persistent mode\\u002fshared memory without kernel config\",\"line\":{\"color\":\"#00cc96\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"Multicore: Persistent mode\\u002fshared memory without kernel config\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0],\"xaxis\":\"x\",\"y\":[90851.4,176159.32,260268.78,336355.99,413750.0,492578.2,565737.17,640579.35,491284.66,540759.63,591144.78,638938.52,687954.18,734886.87,784210.26,834811.24,714178.66,734804.4,740714.93,750425.99,752625.52,760661.34,758401.65,767985.22,785474.61,789679.78,790483.94,798176.69,792763.39,788972.72,789307.69,777440.02,767985.22,775967.55,764863.41,768616.31],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"Configuration=Multicore: afl_execs: Persistent mode\\u002fshared memory + kernel config\\u003cbr\\u003eNumber of parallel fuzzers=%{x}\\u003cbr\\u003eFuzz target executions per second=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Multicore: afl_execs: Persistent mode\\u002fshared memory + kernel config\",\"line\":{\"color\":\"#ab63fa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"Multicore: afl_execs: Persistent mode\\u002fshared memory + kernel config\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0],\"xaxis\":\"x\",\"y\":[134423.5,258490.04,383777.45,496249.48,613089.31,730366.19,844187.32,962846.18,997414.74,1034757.73,1070703.42,1104249.08,1131176.88,1164076.48,1198824.47,1227578.7,1272311.96,1295688.8,1314398.6,1328581.94,1342660.66,1363930.3,1377043.72,1375818.24,1361687.56,1369637.56,1375444.16,1349599.77,1321658.08,1301868.24,1276904.9,1243444.8,1243981.21,1234425.98,1244349.38,1250454.58],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"Configuration=Singlecore: Non-persistent mode + kernel config\\u003cbr\\u003eNumber of parallel fuzzers=%{x}\\u003cbr\\u003eFuzz target executions per second=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Singlecore: Non-persistent mode + kernel config\",\"line\":{\"color\":\"#FFA15A\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"Singlecore: Non-persistent mode + kernel config\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0],\"xaxis\":\"x\",\"y\":[11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96,11038.96],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"Configuration=Singlecore: Persistent mode\\u002fshared memory + kernel config\\u003cbr\\u003eNumber of parallel fuzzers=%{x}\\u003cbr\\u003eFuzz target executions per second=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Singlecore: Persistent mode\\u002fshared memory + kernel config\",\"line\":{\"color\":\"#19d3f3\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"Singlecore: Persistent mode\\u002fshared memory + kernel config\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0],\"xaxis\":\"x\",\"y\":[135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26,135613.26],\"yaxis\":\"y\",\"type\":\"scatter\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Number of parallel fuzzers\"},\"tickvals\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Fuzz target executions per second\"},\"tickvals\":[11019.3,284224.18399999995,557429.068,830633.9519999999,1103838.836,1377043.72],\"ticktext\":[\"1x\",\"26x\",\"51x\",\"75x\",\"100x\",\"125x\"]},\"legend\":{\"title\":{\"text\":\"Configuration\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Fuzzer performance\"},\"width\":1200,\"height\":400}, {\"responsive\": true} ).then(function(){\n", + " \n", + "var gd = document.getElementById('80d54fb8-8ae8-4543-ba3c-c1846c8bbc5b');\n", + "var x = new MutationObserver(function (mutations, observer) {{\n", + " var display = window.getComputedStyle(gd).display;\n", + " if (!display || display === 'none') {{\n", + " console.log([gd, 'removed!']);\n", + " Plotly.purge(gd);\n", + " observer.disconnect();\n", + " }}\n", + "}});\n", + "\n", + "// Listen for the removal of the full notebook cells\n", + "var notebookContainer = gd.closest('#notebook-container');\n", + "if (notebookContainer) {{\n", + " x.observe(notebookContainer, {childList: true});\n", + "}}\n", + "\n", + "// Listen for the clearing of the current output cell\n", + "var outputEl = gd.closest('.output');\n", + "if (outputEl) {{\n", + " x.observe(outputEl, {childList: true});\n", + "}}\n", + "\n", + " }) }; }); </script> </div>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "import numpy as np\n", + "pd.options.plotting.backend = \"plotly\"\n", + "\n", + "# Right now our table has absolute values of execs per sec, but it's more useful\n", + "# to show relative perf (vs 1.0x baseline)\n", + "pivotdf = graphdf.pivot(index=\"parallel_fuzzers\", columns=\"label\", values=\"execs_per_sec\")\n", + "fig = pivotdf.plot(\n", + " title=\"Fuzzer performance\",\n", + " labels={\n", + " \"label\": \"Configuration\",\n", + " \"parallel_fuzzers\": \"Number of parallel fuzzers\",\n", + " \"value\": \"Fuzz target executions per second\"\n", + " }\n", + ")\n", + "\n", + "# Compute tick values and their labels for the primary Y-axis\n", + "tickvals = np.linspace(graphdf['execs_per_sec'].min(), graphdf['execs_per_sec'].max(), 6)\n", + "ticktext = [f\"{val:.0f}x\" for val in tickvals / graphdf['execs_per_sec'].min()]\n", + "# Update the primary Y-axis with custom tick labels\n", + "fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n", + "fig.update_xaxes(tickvals=list(range(1,36+1)))\n", + "fig.update_layout(width=1200, height=400)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's what the table that produced this graph looks like:" + ] + }, + { + "cell_type": "code", + "execution_count": 508, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th>label</th>\n", + " <th>Multicore: Non-persistent mode + kernel config</th>\n", + " <th>Multicore: Persistent mode/shared memory + kernel config</th>\n", + " <th>Multicore: Persistent mode/shared memory without kernel config</th>\n", + " <th>Multicore: afl_execs: Persistent mode/shared memory + kernel config</th>\n", + " <th>Singlecore: Non-persistent mode + kernel config</th>\n", + " <th>Singlecore: Persistent mode/shared memory + kernel config</th>\n", + " </tr>\n", + " <tr>\n", + " <th>parallel_fuzzers</th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>1.0</th>\n", + " <td>11019.30</td>\n", + " <td>133591.26</td>\n", + " <td>90851.40</td>\n", + " <td>134423.50</td>\n", + " <td>11038.96</td>\n", + " <td>135613.26</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2.0</th>\n", + " <td>21111.92</td>\n", + " <td>255995.07</td>\n", + " <td>176159.32</td>\n", + " <td>258490.04</td>\n", + " <td>11038.96</td>\n", + " <td>135613.26</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3.0</th>\n", + " <td>30568.82</td>\n", + " <td>380246.34</td>\n", + " <td>260268.78</td>\n", + " <td>383777.45</td>\n", + " <td>11038.96</td>\n", + " <td>135613.26</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4.0</th>\n", + " <td>38963.07</td>\n", + " <td>490254.72</td>\n", + " <td>336355.99</td>\n", + " <td>496249.48</td>\n", + " <td>11038.96</td>\n", + " <td>135613.26</td>\n", + " </tr>\n", + " <tr>\n", + " <th>5.0</th>\n", + " <td>47693.65</td>\n", + " <td>598698.16</td>\n", + " <td>413750.00</td>\n", + " <td>613089.31</td>\n", + " <td>11038.96</td>\n", + " <td>135613.26</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + "label Multicore: Non-persistent mode + kernel config \\\n", + "parallel_fuzzers \n", + "1.0 11019.30 \n", + "2.0 21111.92 \n", + "3.0 30568.82 \n", + "4.0 38963.07 \n", + "5.0 47693.65 \n", + "\n", + "label Multicore: Persistent mode/shared memory + kernel config \\\n", + "parallel_fuzzers \n", + "1.0 133591.26 \n", + "2.0 255995.07 \n", + "3.0 380246.34 \n", + "4.0 490254.72 \n", + "5.0 598698.16 \n", + "\n", + "label Multicore: Persistent mode/shared memory without kernel config \\\n", + "parallel_fuzzers \n", + "1.0 90851.40 \n", + "2.0 176159.32 \n", + "3.0 260268.78 \n", + "4.0 336355.99 \n", + "5.0 413750.00 \n", + "\n", + "label Multicore: afl_execs: Persistent mode/shared memory + kernel config \\\n", + "parallel_fuzzers \n", + "1.0 134423.50 \n", + "2.0 258490.04 \n", + "3.0 383777.45 \n", + "4.0 496249.48 \n", + "5.0 613089.31 \n", + "\n", + "label Singlecore: Non-persistent mode + kernel config \\\n", + "parallel_fuzzers \n", + "1.0 11038.96 \n", + "2.0 11038.96 \n", + "3.0 11038.96 \n", + "4.0 11038.96 \n", + "5.0 11038.96 \n", + "\n", + "label Singlecore: Persistent mode/shared memory + kernel config \n", + "parallel_fuzzers \n", + "1.0 135613.26 \n", + "2.0 135613.26 \n", + "3.0 135613.26 \n", + "4.0 135613.26 \n", + "5.0 135613.26 " + ] + }, + "execution_count": 508, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pivotdf.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can totally ignore the code cell directly below (unless you're curious). It's just preparing Markdown for the block below it to render. Jupyter Notebooks aren't able to use code variables inside Markdown blocks, so I have to do this instead." + ] + }, + { + "cell_type": "code", + "execution_count": 509, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "\n", + "### Line graph analysis\n", + "Here are a few things that jump out from the graph above. Let's start at the bottom of the graph.\n", + "\n", + "#### test-instr vs. test-instr-persist-shmem\n", + "\n", + "This graph is scaled so that the single-core, non-persistent-mode performance (11038 execs per second) is\n", + "represented as 1.0x. If you build and run a fuzzer without creating a persistent mode harness for it, and without running fuzzers in parallel, this is the performance\n", + "you get on this machine.\n", + "\n", + "#### Multicore test-instr\n", + "\n", + "By running as many parallel fuzzers are there are CPU threads, we can reach 103765 execs per second, which is 9.4x that base speed.\n", + "\n", + "#### Persistent mode + shared memory\n", + "\n", + "By modifying the harness to use persistent mode as described [here](https://github.com/AFLplusplus/AFLplusplus/blob/stable/instrumentation/README.persistent_mode.md#4-persistent-mode),\n", + "we end up with 12.3x base speed. So -- perhaps counter-intuively -- if you have a choice between switching to using multiple cores or rewriting\n", + "the harness to use persistent mode, it is better (at least on this machine) to use persistent mode on a single core, than to use non-persistent mode on every core.\n", + "\n", + "#### Kernel config\n", + "\n", + "By \"kernel config\", I'm referring to booting the Linux kernel with `mitigations=off`, which is a meta-parameter for disabling *all* hardware vulnerability meltdowns (such as Spectre,\n", + "Meltdown, Retbleed, etc) introduced in Linux v5.2. Disabling these results in a `total_execs_per_sec` increase of 368476 execs -- the difference between\n", + "109.0x (mitigations off) and 75.6x (mitigations on) base speed. Turning on mitigations\n", + "reduced the overall performance by 31%!\n", + "\n", + "One way to think about this is that the mitigations turn this 16-thread CPU into a 7-thread CPU, since the number of execs reached with 16 threads and mitigations on is around the same\n", + "number of execs reached with 7 threads and mitigations off.\n", + "\n", + "Or if we want to think in terms of cores, then the average number of execs gained per core in the initial eight is 115588 execs per sec, but the loss due to\n", + "mitigations is 368476 execs per sec, which is the averaged performance of 3.2 cores.\n", + "\n", + "#### afl_execs_per_sec vs. total_execs_per_sec\n", + "\n", + "* The purple line at the top is measuring `afl_execs_per_sec`. This is afl's own measurement of the speed of each fuzzer process, from the `out/fuzzer/fuzzer_stats` file.\n", + " * It peaks at 23 fuzzers running in parallel, on this 8-core (16-thread) CPU.\n", + " * In contrast, `total_execs_per_sec` shows large drops in performance as we pass 8 (cores) and 16 (threads) fuzzers.\n", + " * I'm inclined to trust `total_execs_per_sec` `(total_execs / (end time - start time))` more, so we'll use that from now on.\n", + "\n", + "#### How many parallel fuzzers should we use on this machine?\n", + "\n", + "* The drops in performance after 8/16 fuzzers are profound.\n", + " * Using 9-12 fuzzers is *worse* than using 8 fuzzers on this 8C/16T system, but using 13-16 is better than 8.\n", + " * And using >16 is worse than using 16. Makes sense.\n", + " * We should use the number of CPUs in /proc/cpuinfo (threads) to get the best performance. But if we did halve the number of\n", + " fuzzers, we would surprisingly only lose 21%\n", + " of performance. This could be a good tradeoff in terms of cost.\n" + ], + "text/plain": [ + "<IPython.core.display.Markdown object>" + ] + }, + "execution_count": 509, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# (Ignore this code cell.)\n", + "from IPython.display import Markdown as md\n", + "singlecore_base_execs = pivotdf.iloc[0][\"Singlecore: Non-persistent mode + kernel config\"]\n", + "singlecore_persist_execs = pivotdf.iloc[0][\"Singlecore: Persistent mode/shared memory + kernel config\"]\n", + "multicore_fuzzers_with_afl_max_execs = int(pivotdf[\"Multicore: afl_execs: Persistent mode/shared memory + kernel config\"].idxmax())\n", + "multicore_fuzzers_with_total_max_execs = int(pivotdf[\"Multicore: Persistent mode/shared memory + kernel config\"].idxmax())\n", + "multicore_base_max_execs = pivotdf[\"Multicore: Non-persistent mode + kernel config\"].max()\n", + "factor_for_execs = lambda execs: round(execs / singlecore_base_execs, 1)\n", + "\n", + "multicore_persistent_without_mitigations_label = \"Multicore: Persistent mode/shared memory + kernel config\"\n", + "multicore_max_execs_mitigations_off = pivotdf[multicore_persistent_without_mitigations_label].max()\n", + "multicore_max_execs_mitigations_off_only_cores = pivotdf.loc[multicore_fuzzers_with_total_max_execs / 2][multicore_persistent_without_mitigations_label]\n", + "multicore_max_execs_mitigations_on = pivotdf[\"Multicore: Persistent mode/shared memory without kernel config\"].max()\n", + "multicore_avg_gain_per_core = pivotdf.loc[pivotdf.index <= 8][\"Multicore: Persistent mode/shared memory + kernel config\"].diff().dropna().mean()\n", + "mitigations_off_increase = int(multicore_max_execs_mitigations_off - multicore_max_execs_mitigations_on)\n", + "\n", + "md(f\"\"\"\n", + "### Line graph analysis\n", + "Here are a few things that jump out from the graph above. Let's start at the bottom of the graph.\n", + "\n", + "#### test-instr vs. test-instr-persist-shmem\n", + "\n", + "This graph is scaled so that the single-core, non-persistent-mode performance ({int(singlecore_base_execs)} execs per second) is\n", + "represented as 1.0x. If you build and run a fuzzer without creating a persistent mode harness for it, and without running fuzzers in parallel, this is the performance\n", + "you get on this machine.\n", + "\n", + "#### Multicore test-instr\n", + "\n", + "By running as many parallel fuzzers are there are CPU threads, we can reach {int(multicore_base_max_execs)} execs per second, which is {factor_for_execs(multicore_base_max_execs)}x that base speed.\n", + "\n", + "#### Persistent mode + shared memory\n", + "\n", + "By modifying the harness to use persistent mode as described [here](https://github.com/AFLplusplus/AFLplusplus/blob/stable/instrumentation/README.persistent_mode.md#4-persistent-mode),\n", + "we end up with {factor_for_execs(singlecore_persist_execs)}x base speed. So -- perhaps counter-intuively -- if you have a choice between switching to using multiple cores or rewriting\n", + "the harness to use persistent mode, it is better (at least on this machine) to use persistent mode on a single core, than to use non-persistent mode on every core.\n", + "\n", + "#### Kernel config\n", + "\n", + "By \"kernel config\", I'm referring to booting the Linux kernel with `mitigations=off`, which is a meta-parameter for disabling *all* hardware vulnerability meltdowns (such as Spectre,\n", + "Meltdown, Retbleed, etc) introduced in Linux v5.2. Disabling these results in a `total_execs_per_sec` increase of {mitigations_off_increase} execs -- the difference between\n", + "{factor_for_execs(multicore_max_execs_mitigations_off)}x (mitigations off) and {factor_for_execs(multicore_max_execs_mitigations_on)}x (mitigations on) base speed. Turning on mitigations\n", + "reduced the overall performance by {abs(round(((multicore_max_execs_mitigations_on - multicore_max_execs_mitigations_off) / multicore_max_execs_mitigations_off) * 100))}%!\n", + "\n", + "One way to think about this is that the mitigations turn this 16-thread CPU into a 7-thread CPU, since the number of execs reached with 16 threads and mitigations on is around the same\n", + "number of execs reached with 7 threads and mitigations off.\n", + "\n", + "Or if we want to think in terms of cores, then the average number of execs gained per core in the initial eight is {int(multicore_avg_gain_per_core)} execs per sec, but the loss due to\n", + "mitigations is {mitigations_off_increase} execs per sec, which is the averaged performance of {round(mitigations_off_increase / multicore_avg_gain_per_core, 1)} cores.\n", + "\n", + "#### afl_execs_per_sec vs. total_execs_per_sec\n", + "\n", + "* The purple line at the top is measuring `afl_execs_per_sec`. This is afl's own measurement of the speed of each fuzzer process, from the `out/fuzzer/fuzzer_stats` file.\n", + " * It peaks at {multicore_fuzzers_with_afl_max_execs} fuzzers running in parallel, on this 8-core (16-thread) CPU.\n", + " * In contrast, `total_execs_per_sec` shows large drops in performance as we pass 8 (cores) and 16 (threads) fuzzers.\n", + " * I'm inclined to trust `total_execs_per_sec` `(total_execs / (end time - start time))` more, so we'll use that from now on.\n", + "\n", + "#### How many parallel fuzzers should we use on this machine?\n", + "\n", + "* The drops in performance after 8/16 fuzzers are profound.\n", + " * Using 9-12 fuzzers is *worse* than using 8 fuzzers on this 8C/16T system, but using 13-16 is better than 8.\n", + " * And using >16 is worse than using 16. Makes sense.\n", + " * We should use the number of CPUs in /proc/cpuinfo (threads) to get the best performance. But if we did halve the number of\n", + " fuzzers, we would surprisingly only lose {abs(int(((multicore_max_execs_mitigations_off_only_cores - multicore_max_execs_mitigations_off) / multicore_max_execs_mitigations_off) * 100))}%\n", + " of performance. This could be a good tradeoff in terms of cost.\n", + "\"\"\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Example with more cores\n", + "\n", + "While there was some nuance here, the answer was pretty straightforward -- use the number of CPU threads you have access to. What if there were more threads? Here the experiment is repeated on an AWS EC2 \"r6a.48xlarge\" spot instance with 192 vCPUs, and the answer calls the conclusion we just made above into question:" + ] + }, + { + "cell_type": "code", + "execution_count": 521, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>config.afl_persistent_config</th>\n", + " <th>config.afl_system_config</th>\n", + " <th>config.afl_version</th>\n", + " <th>config.comment</th>\n", + " <th>config.compiler</th>\n", + " <th>config.target_arch</th>\n", + " <th>hardware.cpu_fastest_core_mhz</th>\n", + " <th>hardware.cpu_model</th>\n", + " <th>hardware.cpu_threads</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.afl_execs_per_sec</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.afl_execs_total</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.fuzzers_used</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.run_end</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.run_start</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.total_execs_per_sec</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.total_run_time</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>148</th>\n", + " <td>False</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>AWS EC2 r6a.48xlarge spot instance</td>\n", + " <td>clang version 15.0.7 (Amazon Linux 15.0.7-3.am...</td>\n", + " <td>x86_64-amazon-linux-gnu</td>\n", + " <td>3599.314</td>\n", + " <td>AMD EPYC 7R13 Processor</td>\n", + " <td>192</td>\n", + " <td>85586.47</td>\n", + " <td>519670.0</td>\n", + " <td>1.0</td>\n", + " <td>2023-09-30 07:42:00.479418</td>\n", + " <td>2023-09-30 07:41:57.396293</td>\n", + " <td>84636.81</td>\n", + " <td>6.14</td>\n", + " </tr>\n", + " <tr>\n", + " <th>149</th>\n", + " <td>False</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>AWS EC2 r6a.48xlarge spot instance</td>\n", + " <td>clang version 15.0.7 (Amazon Linux 15.0.7-3.am...</td>\n", + " <td>x86_64-amazon-linux-gnu</td>\n", + " <td>3599.425</td>\n", + " <td>AMD EPYC 7R13 Processor</td>\n", + " <td>192</td>\n", + " <td>171655.96</td>\n", + " <td>1039340.0</td>\n", + " <td>2.0</td>\n", + " <td>2023-09-30 07:42:06.853436</td>\n", + " <td>2023-09-30 07:42:03.776562</td>\n", + " <td>168998.37</td>\n", + " <td>6.15</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " config.afl_persistent_config config.afl_system_config \\\n", + "148 False True \n", + "149 False True \n", + "\n", + " config.afl_version config.comment \\\n", + "148 ++4.09a AWS EC2 r6a.48xlarge spot instance \n", + "149 ++4.09a AWS EC2 r6a.48xlarge spot instance \n", + "\n", + " config.compiler \\\n", + "148 clang version 15.0.7 (Amazon Linux 15.0.7-3.am... \n", + "149 clang version 15.0.7 (Amazon Linux 15.0.7-3.am... \n", + "\n", + " config.target_arch hardware.cpu_fastest_core_mhz \\\n", + "148 x86_64-amazon-linux-gnu 3599.314 \n", + "149 x86_64-amazon-linux-gnu 3599.425 \n", + "\n", + " hardware.cpu_model hardware.cpu_threads \\\n", + "148 AMD EPYC 7R13 Processor 192 \n", + "149 AMD EPYC 7R13 Processor 192 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.afl_execs_per_sec \\\n", + "148 85586.47 \n", + "149 171655.96 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.afl_execs_total \\\n", + "148 519670.0 \n", + "149 1039340.0 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.fuzzers_used \\\n", + "148 1.0 \n", + "149 2.0 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.run_end \\\n", + "148 2023-09-30 07:42:00.479418 \n", + "149 2023-09-30 07:42:06.853436 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.run_start \\\n", + "148 2023-09-30 07:41:57.396293 \n", + "149 2023-09-30 07:42:03.776562 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.total_execs_per_sec \\\n", + "148 84636.81 \n", + "149 168998.37 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.total_run_time \n", + "148 6.14 \n", + "149 6.15 " + ] + }, + "execution_count": 521, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r6a = df.query(\"`config.comment` == 'AWS EC2 r6a.48xlarge spot instance'\")\n", + "r6a.head(2).dropna(axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 516, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>execs_per_sec</th>\n", + " <th>parallel_fuzzers</th>\n", + " <th>afl_persistent_config</th>\n", + " <th>afl_system_config</th>\n", + " <th>label</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>399</th>\n", + " <td>331957.22</td>\n", + " <td>200.0</td>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>Multicore: afl_execs: Persistent mode/shared m...</td>\n", + " </tr>\n", + " <tr>\n", + " <th>153</th>\n", + " <td>1026766.44</td>\n", + " <td>77.0</td>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>Multicore: afl_execs: Persistent mode/shared m...</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " execs_per_sec parallel_fuzzers afl_persistent_config \\\n", + "399 331957.22 200.0 True \n", + "153 1026766.44 77.0 True \n", + "\n", + " afl_system_config label \n", + "399 True Multicore: afl_execs: Persistent mode/shared m... \n", + "153 True Multicore: afl_execs: Persistent mode/shared m... " + ] + }, + "execution_count": 516, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r6a_graphdf = build_graphdf_from_query(r6a)\n", + "r6a_graphdf.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 514, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "hovertemplate": "Configuration=Multicore: Persistent mode/shared memory + kernel config<br>Number of parallel fuzzers=%{x}<br>Fuzz target executions per second=%{y}<extra></extra>", + "legendgroup": "Multicore: Persistent mode/shared memory + kernel config", + "line": { + "color": "#636efa", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "Multicore: Persistent mode/shared memory + kernel config", + "orientation": "v", + "showlegend": true, + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200 + ], + "xaxis": "x", + "y": [ + 105839.1, + 210819.47, + 316229.21, + 413256.46, + 518632.73, + 619884.69, + 702256.76, + 804131.53, + 847288.04, + 943139.75, + 1024439.07, + 1113578.57, + 1162772.81, + 1265283.48, + 1346295.34, + 1421319.66, + 1389055.03, + 1336294.29, + 1354421.12, + 1341083.87, + 1267487.8, + 1228006.44, + 1164952.24, + 1117569.89, + 1027015.81, + 1011333.83, + 993703.26, + 983824.21, + 969783.14, + 953522.94, + 937704.89, + 914207.81, + 898800.31, + 896437.34, + 895982.76, + 896412.07, + 888119.63, + 874168.22, + 858049.53, + 839870.71, + 830337.88, + 833695.19, + 832246.18, + 831472, + 820819.59, + 809235.61, + 796104.63, + 779748.67, + 651915.77, + 658310.11, + 664906.42, + 670874.88, + 679223.43, + 680460.23, + 683449.31, + 683776.32, + 642820.96, + 656234.7, + 664079.06, + 668815.96, + 677202.95, + 682183.78, + 687798.53, + 695065.41, + 653862.76, + 662255.65, + 667137.25, + 674252.38, + 674769.27, + 676779.76, + 678994.74, + 677707.66, + 643636.07, + 645011.41, + 647860.09, + 650443.61, + 655762.21, + 655894.58, + 659395.36, + 660632.48, + 636619.43, + 636679.37, + 641091.3, + 642134.44, + 644188.04, + 645646.45, + 648190.91, + 645188.95, + 623576.69, + 623521.92, + 627188.81, + 631735.31, + 632169.99, + 632104.98, + 636441.28, + 636413.09, + 627747.11, + 618580.91, + 610289.87, + 602517.1, + 587297.53, + 574782.55, + 559428.41, + 543938.93, + 538120.81, + 531555.69, + 526211.57, + 521796.77, + 510905.75, + 501877.11, + 490630.76, + 479788.36, + 475372.8, + 470776.14, + 466163.29, + 463421.1, + 454522.49, + 446589.03, + 438649.5, + 428977.64, + 424292.46, + 418867.74, + 414980.74, + 412384.2, + 404224.17, + 396310.94, + 389392.67, + 381715.99, + 377825.09, + 374215.78, + 370708.15, + 368402.32, + 361940.65, + 355502.6, + 349589.01, + 342501.52, + 339897.15, + 337989.11, + 335723.22, + 334039.83, + 329127.04, + 323073.5, + 318652.21, + 313198.69, + 311591.71, + 310530.42, + 309390.36, + 307977.56, + 306559.97, + 305066.46, + 302157.29, + 301097.52, + 300138.11, + 299824.88, + 298130.63, + 297838.27, + 296189.21, + 294695.11, + 292518.21, + 291030.08, + 290207.87, + 289828.9, + 289247.99, + 288295.58, + 287447.58, + 286405.31, + 284755.57, + 284046.79, + 283176.27, + 283053.87, + 282178.36, + 281522.15, + 280377.23, + 279885.55, + 278793.08, + 277923.89, + 277069.78, + 277124.24, + 276576.49, + 276020.64, + 275328, + 275029.74, + 274030.79, + 273612.38, + 273285.13, + 273039.33, + 272819.57, + 272960.42, + 272388.01, + 272311.26, + 272115.97, + 272056.42, + 271835.4, + 271397.63, + 271867.2, + 271065.21, + 270797.96, + 270150.29, + 269442.01, + 268674.91 + ], + "yaxis": "y" + }, + { + "hovertemplate": "Configuration=Multicore: afl_execs: Persistent mode/shared memory + kernel config<br>Number of parallel fuzzers=%{x}<br>Fuzz target executions per second=%{y}<extra></extra>", + "legendgroup": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", + "line": { + "color": "#EF553B", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "Multicore: afl_execs: Persistent mode/shared memory + kernel config", + "orientation": "v", + "showlegend": true, + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200 + ], + "xaxis": "x", + "y": [ + 107126.58, + 214213.66, + 322468.69, + 427406.37, + 535728.44, + 643227.5, + 746997.96, + 852324.44, + 898199.42, + 994921.42, + 1086491.72, + 1188114.32, + 1275638.92, + 1373504.32, + 1456611.99, + 1547050.37, + 1556304.25, + 1498337.65, + 1480610.39, + 1442181.26, + 1380390.08, + 1315149.42, + 1250840.52, + 1198962.91, + 1113901.96, + 1112866.02, + 1109572.68, + 1112386.81, + 1104839.85, + 1088259.95, + 1072951.12, + 1038335.3, + 1039005.59, + 1055683.11, + 1074708.24, + 1088882.64, + 1084369.02, + 1061476.09, + 1037330.81, + 1001283.07, + 1011982.42, + 1039061.07, + 1060191.68, + 1069379.92, + 1051676.06, + 1025702.93, + 1000795.88, + 959941, + 928333.9, + 936392.44, + 947163.68, + 958614.58, + 973982.54, + 976113.12, + 983432.87, + 985159.38, + 949664.42, + 960540.52, + 971717.37, + 978223.94, + 995090.76, + 1000123.55, + 1006856.18, + 1013280.29, + 977531.19, + 988260.54, + 996765.65, + 1006933, + 1016151.03, + 1020419.88, + 1024544.66, + 1027862.2, + 989415.52, + 999208.44, + 1009747.84, + 1016122.1, + 1026766.44, + 1032416.84, + 1037369.06, + 1037677.89, + 1001527.34, + 1008569.78, + 1024112.93, + 1033177.84, + 1035389.26, + 1040484.52, + 1047416.67, + 1043614.54, + 1014160.19, + 1019409.94, + 1033667.5, + 1040422.32, + 1045409.98, + 1048162.62, + 1050384.15, + 1050304.88, + 1037251.1, + 1023279.61, + 1009889.86, + 996157.16, + 973425.48, + 960922.5, + 941705.52, + 927206.03, + 919716.12, + 907116.8, + 898444.05, + 889678.68, + 871535.65, + 858369.28, + 839357.6, + 828077.49, + 817619.99, + 806563.34, + 795820.84, + 789602.32, + 769744.98, + 754704.16, + 739965.24, + 721357.74, + 705584.89, + 689179.3, + 674153.86, + 668264.05, + 648129.94, + 630733.08, + 614518.38, + 598284.67, + 580642.38, + 562735.32, + 547668.6, + 540727.65, + 519637, + 499189.04, + 482457.86, + 458655.34, + 453087.56, + 445650.76, + 438779.54, + 434421.26, + 422130, + 403403.62, + 391528.74, + 374715.06, + 372678.44, + 371466.33, + 369815.4, + 367734.06, + 366332.54, + 365256.89, + 362078.84, + 361083.46, + 359994.43, + 359950.89, + 357498.64, + 357285.14, + 355405.08, + 354127.44, + 351793.59, + 350348, + 349438.44, + 349188.38, + 348377.38, + 347124.06, + 346480.82, + 345660.61, + 344352.86, + 343903.25, + 342402.74, + 342935.7, + 342089.26, + 341369.47, + 340166.19, + 339692.96, + 339204.8, + 338925.12, + 337700.46, + 338203.76, + 337556.9, + 336873.92, + 336399.84, + 336455.79, + 335823.56, + 335587.52, + 335620.09, + 334996.68, + 334980.98, + 335404.84, + 335051.8, + 334887.42, + 335150.96, + 334773.71, + 335035.28, + 334596.91, + 336065.8, + 335034.33, + 334931.36, + 334191.98, + 332929.11, + 331957.22 + ], + "yaxis": "y" + } + ], + "layout": { + "height": 400, + "legend": { + "title": { + "text": "Configuration" + }, + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Fuzzer performance" + }, + "width": 1200, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "tickvals": [ + 0, + 5, + 10, + 15, + 20, + 25, + 30, + 35, + 40, + 45, + 50, + 55, + 60, + 65, + 70, + 75, + 80, + 85, + 90, + 95, + 100, + 105, + 110, + 115, + 120, + 125, + 130, + 135, + 140, + 145, + 150, + 155, + 160, + 165, + 170, + 175, + 180, + 185, + 190, + 195, + 200 + ], + "title": { + "text": "Number of parallel fuzzers" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "ticktext": [ + "10x", + "36x", + "62x", + "89x", + "115x", + "141x" + ], + "tickvals": [ + 105839.1, + 395932.13, + 686025.1599999999, + 976118.1899999998, + 1266211.22, + 1556304.25 + ], + "title": { + "text": "Fuzz target executions per second" + } + } + } + }, + "text/html": [ + "<div> <div id=\"12f69392-4283-47e6-a491-90bdd9508929\" class=\"plotly-graph-div\" style=\"height:400px; width:1200px;\"></div> <script type=\"text/javascript\"> require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"12f69392-4283-47e6-a491-90bdd9508929\")) { Plotly.newPlot( \"12f69392-4283-47e6-a491-90bdd9508929\", [{\"hovertemplate\":\"Configuration=Multicore: Persistent mode\\u002fshared memory + kernel config\\u003cbr\\u003eNumber of parallel fuzzers=%{x}\\u003cbr\\u003eFuzz target executions per second=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Multicore: Persistent mode\\u002fshared memory + kernel config\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"Multicore: Persistent mode\\u002fshared memory + kernel config\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0,51.0,52.0,53.0,54.0,55.0,56.0,57.0,58.0,59.0,60.0,61.0,62.0,63.0,64.0,65.0,66.0,67.0,68.0,69.0,70.0,71.0,72.0,73.0,74.0,75.0,76.0,77.0,78.0,79.0,80.0,81.0,82.0,83.0,84.0,85.0,86.0,87.0,88.0,89.0,90.0,91.0,92.0,93.0,94.0,95.0,96.0,97.0,98.0,99.0,100.0,101.0,102.0,103.0,104.0,105.0,106.0,107.0,108.0,109.0,110.0,111.0,112.0,113.0,114.0,115.0,116.0,117.0,118.0,119.0,120.0,121.0,122.0,123.0,124.0,125.0,126.0,127.0,128.0,129.0,130.0,131.0,132.0,133.0,134.0,135.0,136.0,137.0,138.0,139.0,140.0,141.0,142.0,143.0,144.0,145.0,146.0,147.0,148.0,149.0,150.0,151.0,152.0,153.0,154.0,155.0,156.0,157.0,158.0,159.0,160.0,161.0,162.0,163.0,164.0,165.0,166.0,167.0,168.0,169.0,170.0,171.0,172.0,173.0,174.0,175.0,176.0,177.0,178.0,179.0,180.0,181.0,182.0,183.0,184.0,185.0,186.0,187.0,188.0,189.0,190.0,191.0,192.0,193.0,194.0,195.0,196.0,197.0,198.0,199.0,200.0],\"xaxis\":\"x\",\"y\":[105839.1,210819.47,316229.21,413256.46,518632.73,619884.69,702256.76,804131.53,847288.04,943139.75,1024439.07,1113578.57,1162772.81,1265283.48,1346295.34,1421319.66,1389055.03,1336294.29,1354421.12,1341083.87,1267487.8,1228006.44,1164952.24,1117569.89,1027015.81,1011333.83,993703.26,983824.21,969783.14,953522.94,937704.89,914207.81,898800.31,896437.34,895982.76,896412.07,888119.63,874168.22,858049.53,839870.71,830337.88,833695.19,832246.18,831472.0,820819.59,809235.61,796104.63,779748.67,651915.77,658310.11,664906.42,670874.88,679223.43,680460.23,683449.31,683776.32,642820.96,656234.7,664079.06,668815.96,677202.95,682183.78,687798.53,695065.41,653862.76,662255.65,667137.25,674252.38,674769.27,676779.76,678994.74,677707.66,643636.07,645011.41,647860.09,650443.61,655762.21,655894.58,659395.36,660632.48,636619.43,636679.37,641091.3,642134.44,644188.04,645646.45,648190.91,645188.95,623576.69,623521.92,627188.81,631735.31,632169.99,632104.98,636441.28,636413.09,627747.11,618580.91,610289.87,602517.1,587297.53,574782.55,559428.41,543938.93,538120.81,531555.69,526211.57,521796.77,510905.75,501877.11,490630.76,479788.36,475372.8,470776.14,466163.29,463421.1,454522.49,446589.03,438649.5,428977.64,424292.46,418867.74,414980.74,412384.2,404224.17,396310.94,389392.67,381715.99,377825.09,374215.78,370708.15,368402.32,361940.65,355502.6,349589.01,342501.52,339897.15,337989.11,335723.22,334039.83,329127.04,323073.5,318652.21,313198.69,311591.71,310530.42,309390.36,307977.56,306559.97,305066.46,302157.29,301097.52,300138.11,299824.88,298130.63,297838.27,296189.21,294695.11,292518.21,291030.08,290207.87,289828.9,289247.99,288295.58,287447.58,286405.31,284755.57,284046.79,283176.27,283053.87,282178.36,281522.15,280377.23,279885.55,278793.08,277923.89,277069.78,277124.24,276576.49,276020.64,275328.0,275029.74,274030.79,273612.38,273285.13,273039.33,272819.57,272960.42,272388.01,272311.26,272115.97,272056.42,271835.4,271397.63,271867.2,271065.21,270797.96,270150.29,269442.01,268674.91],\"yaxis\":\"y\",\"type\":\"scatter\"},{\"hovertemplate\":\"Configuration=Multicore: afl_execs: Persistent mode\\u002fshared memory + kernel config\\u003cbr\\u003eNumber of parallel fuzzers=%{x}\\u003cbr\\u003eFuzz target executions per second=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"Multicore: afl_execs: Persistent mode\\u002fshared memory + kernel config\",\"line\":{\"color\":\"#EF553B\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"Multicore: afl_execs: Persistent mode\\u002fshared memory + kernel config\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0,51.0,52.0,53.0,54.0,55.0,56.0,57.0,58.0,59.0,60.0,61.0,62.0,63.0,64.0,65.0,66.0,67.0,68.0,69.0,70.0,71.0,72.0,73.0,74.0,75.0,76.0,77.0,78.0,79.0,80.0,81.0,82.0,83.0,84.0,85.0,86.0,87.0,88.0,89.0,90.0,91.0,92.0,93.0,94.0,95.0,96.0,97.0,98.0,99.0,100.0,101.0,102.0,103.0,104.0,105.0,106.0,107.0,108.0,109.0,110.0,111.0,112.0,113.0,114.0,115.0,116.0,117.0,118.0,119.0,120.0,121.0,122.0,123.0,124.0,125.0,126.0,127.0,128.0,129.0,130.0,131.0,132.0,133.0,134.0,135.0,136.0,137.0,138.0,139.0,140.0,141.0,142.0,143.0,144.0,145.0,146.0,147.0,148.0,149.0,150.0,151.0,152.0,153.0,154.0,155.0,156.0,157.0,158.0,159.0,160.0,161.0,162.0,163.0,164.0,165.0,166.0,167.0,168.0,169.0,170.0,171.0,172.0,173.0,174.0,175.0,176.0,177.0,178.0,179.0,180.0,181.0,182.0,183.0,184.0,185.0,186.0,187.0,188.0,189.0,190.0,191.0,192.0,193.0,194.0,195.0,196.0,197.0,198.0,199.0,200.0],\"xaxis\":\"x\",\"y\":[107126.58,214213.66,322468.69,427406.37,535728.44,643227.5,746997.96,852324.44,898199.42,994921.42,1086491.72,1188114.32,1275638.92,1373504.32,1456611.99,1547050.37,1556304.25,1498337.65,1480610.39,1442181.26,1380390.08,1315149.42,1250840.52,1198962.91,1113901.96,1112866.02,1109572.68,1112386.81,1104839.85,1088259.95,1072951.12,1038335.3,1039005.59,1055683.11,1074708.24,1088882.64,1084369.02,1061476.09,1037330.81,1001283.07,1011982.42,1039061.07,1060191.68,1069379.92,1051676.06,1025702.93,1000795.88,959941.0,928333.9,936392.44,947163.68,958614.58,973982.54,976113.12,983432.87,985159.38,949664.42,960540.52,971717.37,978223.94,995090.76,1000123.55,1006856.18,1013280.29,977531.19,988260.54,996765.65,1006933.0,1016151.03,1020419.88,1024544.66,1027862.2,989415.52,999208.44,1009747.84,1016122.1,1026766.44,1032416.84,1037369.06,1037677.89,1001527.34,1008569.78,1024112.93,1033177.84,1035389.26,1040484.52,1047416.67,1043614.54,1014160.19,1019409.94,1033667.5,1040422.32,1045409.98,1048162.62,1050384.15,1050304.88,1037251.1,1023279.61,1009889.86,996157.16,973425.48,960922.5,941705.52,927206.03,919716.12,907116.8,898444.05,889678.68,871535.65,858369.28,839357.6,828077.49,817619.99,806563.34,795820.84,789602.32,769744.98,754704.16,739965.24,721357.74,705584.89,689179.3,674153.86,668264.05,648129.94,630733.08,614518.38,598284.67,580642.38,562735.32,547668.6,540727.65,519637.0,499189.04,482457.86,458655.34,453087.56,445650.76,438779.54,434421.26,422130.0,403403.62,391528.74,374715.06,372678.44,371466.33,369815.4,367734.06,366332.54,365256.89,362078.84,361083.46,359994.43,359950.89,357498.64,357285.14,355405.08,354127.44,351793.59,350348.0,349438.44,349188.38,348377.38,347124.06,346480.82,345660.61,344352.86,343903.25,342402.74,342935.7,342089.26,341369.47,340166.19,339692.96,339204.8,338925.12,337700.46,338203.76,337556.9,336873.92,336399.84,336455.79,335823.56,335587.52,335620.09,334996.68,334980.98,335404.84,335051.8,334887.42,335150.96,334773.71,335035.28,334596.91,336065.8,335034.33,334931.36,334191.98,332929.11,331957.22],\"yaxis\":\"y\",\"type\":\"scatter\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Number of parallel fuzzers\"},\"tickvals\":[0,5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Fuzz target executions per second\"},\"tickvals\":[105839.1,395932.13,686025.1599999999,976118.1899999998,1266211.22,1556304.25],\"ticktext\":[\"10x\",\"36x\",\"62x\",\"89x\",\"115x\",\"141x\"]},\"legend\":{\"title\":{\"text\":\"Configuration\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Fuzzer performance\"},\"width\":1200,\"height\":400}, {\"responsive\": true} ).then(function(){\n", + " \n", + "var gd = document.getElementById('12f69392-4283-47e6-a491-90bdd9508929');\n", + "var x = new MutationObserver(function (mutations, observer) {{\n", + " var display = window.getComputedStyle(gd).display;\n", + " if (!display || display === 'none') {{\n", + " console.log([gd, 'removed!']);\n", + " Plotly.purge(gd);\n", + " observer.disconnect();\n", + " }}\n", + "}});\n", + "\n", + "// Listen for the removal of the full notebook cells\n", + "var notebookContainer = gd.closest('#notebook-container');\n", + "if (notebookContainer) {{\n", + " x.observe(notebookContainer, {childList: true});\n", + "}}\n", + "\n", + "// Listen for the clearing of the current output cell\n", + "var outputEl = gd.closest('.output');\n", + "if (outputEl) {{\n", + " x.observe(outputEl, {childList: true});\n", + "}}\n", + "\n", + " }) }; }); </script> </div>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "r6a_pivotdf = r6a_graphdf.pivot(index=\"parallel_fuzzers\", columns=\"label\", values=\"execs_per_sec\")\n", + "r6a_fig = r6a_pivotdf.plot(\n", + " title=\"Fuzzer performance\",\n", + " labels={\n", + " \"label\": \"Configuration\",\n", + " \"parallel_fuzzers\": \"Number of parallel fuzzers\",\n", + " \"value\": \"Fuzz target executions per second\"\n", + " }\n", + ")\n", + "\n", + "# Compute tick values and their labels for the primary Y-axis\n", + "tickvals = np.linspace(r6a_graphdf['execs_per_sec'].min(), r6a_graphdf['execs_per_sec'].max(), 6)\n", + "ticktext = [f\"{val:.0f}x\" for val in tickvals / graphdf['execs_per_sec'].min()]\n", + "# Update the primary Y-axis with custom tick labels\n", + "r6a_fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n", + "r6a_fig.update_xaxes(tickvals=list(range(0,200+1, 5)))\n", + "r6a_fig.update_layout(width=1200, height=400)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Line graph analysis\n", + "\n", + "This is a shocking result for a 192 vCPU machine -- whether you count `afl_execs` or `total_execs`, our optimal number of parallel fuzzers was 16!\n", + "\n", + "Does this mean that AFL++ is a bad fuzzer, or that AWS tricked us and gave us a 16-thread machine instead of a 192-thread one?\n", + "\n", + "No -- the most likely cause here (based on a tip from @eqv) is that we're actually saturating the Linux kernel's ability to service system calls. We could look to reduce these, but there's another option available to us, which is to try running more system-call-servicers (read: kernels) at once, on this machine. One way to do that is to use hardware virtualization with KVM,\n", + "and if it is true that this particular fuzzer setup bottlenecks around 16 fuzzers, then we might expect an optimal number of KVM\n", + "kernels running on this machine to be around 16/192 == 8, each with 16 fuzzers in parallel, and perhaps a shared (in-memory?)\n", + "filesystem for the fuzzing queue." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Measuring system call saturation and experimenting with extra KVM hosts\n", + "\n", + "Coming soon!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} |