From c40901318418f924733652714904f75e7eb7cc75 Mon Sep 17 00:00:00 2001 From: chrisholder Date: Tue, 12 Aug 2025 15:30:08 +0100 Subject: [PATCH 01/10] initial setup for benchmarking --- .gitignore | 2 + .pre-commit-config.yaml | 1 + benchmarks/README.md | 74 ++++++++++++++++ benchmarks/asv.conf.json | 88 ++++++++++++++++++ benchmarks/benchmarks/__init__.py | 0 benchmarks/benchmarks/common.py | 24 +++++ benchmarks/benchmarks/distances.py | 118 +++++++++++++++++++++++++ benchmarks/process_global_bencmarks.py | 49 ++++++++++ pyproject.toml | 1 + 9 files changed, 357 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/asv.conf.json create mode 100644 benchmarks/benchmarks/__init__.py create mode 100644 benchmarks/benchmarks/common.py create mode 100644 benchmarks/benchmarks/distances.py create mode 100644 benchmarks/process_global_bencmarks.py diff --git a/.gitignore b/.gitignore index b22a5e4d3d..7bd8adf94a 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,5 @@ local_code/ local/ local_code.py local.py + +/benchmarks/benchmark_results/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 847bcf11b7..d68ef8bd26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,6 +31,7 @@ repos: hooks: - id: ruff args: [ "--fix"] + exclude: '(^|/)benchmarks/' - repo: https://github.com/asottile/pyupgrade rev: v3.20.0 diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000000..40607a227b --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,74 @@ +[//]: # (This was adapted from: https://github.com/scipy/scipy/tree/main/benchmarks) +# aeon Time Series Benchmarks + +Benchmarking aeon with Airspeed Velocity. + +## Usage + +Airspeed Velocity manages building and Python environments by itself, unless told +otherwise. Some of the benchmarking features in `spin` also tell ASV to use the aeon +compiled by `spin`. To run the benchmarks, you will need to install the "dev" +dependencies of aeon: + +```bash +pip install --editble .[dev] +# NOTE: If the above fails, try running pip install --editable ".[dev]" +``` + +Run a benchmark against currently checked-out aeon version (don't record the result): + +```bash +spin bench --submodule classification.distance_based +``` + +Compare change in benchmark results with another branch: + +```bash +spin bench --compare main --submodule classification.distance_based +``` + +Run ASV commands directly (note, this will not set env vars for `ccache` and disabling BLAS/LAPACK multi-threading, as `spin` does): + +```bash +cd benchmarks +asv run --skip-existing-commits --steps 10 ALL +asv publish +asv preview +``` + +More on how to use `asv` can be found in [ASV documentation](https://asv.readthedocs.io/). Command-line help is available as usual via `asv --help` and `asv run --help`. + +## Writing benchmarks + +See [ASV documentation](https://asv.readthedocs.io/) for the basics on how to write benchmarks. + +Some things to consider: + +- When importing things from aeon on the top of the test files, do it as: + + ```python + from .common import safe_import + + with safe_import(): + from aeon.classification.distance_based import KNeighborsTimeSeriesClassifier + ``` + + The benchmark files need to be importable also when benchmarking old versions of aeon. The benchmarks themselves don't need any guarding against missing features — only the top-level imports. + +- Try to keep the runtime of the benchmark reasonable. + +- Use ASV's `time_` methods for benchmarking times rather than cooking up time measurements via `time.clock`, even if it requires some juggling when writing the benchmark. + +- Preparing arrays etc., should generally be put in the `setup` method rather than the `time_` methods, to avoid counting preparation time together with the time of the benchmarked operation. + +- Use `run_monitored` from `common.py` if you need to measure memory usage. + +- Benchmark versioning: by default `asv` invalidates old results when there is any code change in the benchmark routine or in setup/setup_cache. + + This can be controlled manually by setting a fixed benchmark version number, using the `version` attribute. See [ASV documentation](https://asv.readthedocs.io/) for details. + + If set manually, the value needs to be changed manually when old results should be invalidated. In case you want to preserve previous benchmark results when the benchmark did not previously have a manual `version` attribute, the automatically computed default values can be found in `results/benchmark.json`. + +- Benchmark attributes such as `params` and `param_names` must be the same regardless of whether some features are available, or e.g. AEON_XSLOW=1 is set. + + Instead, benchmarks that should not be run can be skipped by raising `NotImplementedError` in `setup()`. diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json new file mode 100644 index 0000000000..994e73c1e6 --- /dev/null +++ b/benchmarks/asv.conf.json @@ -0,0 +1,88 @@ +// This file was taken and adapted from: https://github.com/scipy/scipy/blob/main/benchmarks/asv.conf.json +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "aeon", + + // The project's homepage + "project_url": "https://www.aeon-toolkit.org", + + // The URL of the source code repository for the project being + // benchmarked + "repo": "..", + "dvcs": "git", + "branches": ["HEAD"], + + // Customizable commands for building, installing, and + // uninstalling the project. See asv.conf.json documentation. + // + // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + // "build_command": [ + // "PIP_NO_BUILD_ISOLATION=false python -m pip install . --no-deps --no-index -w {build_cache_dir} {build_dir}" + // ], + + "build_command": [ + "python -m build --wheel -o {build_cache_dir} {build_dir}" + ], + + // The base URL to show a commit for the project. + "show_commit_url": "https://github.com/aeon-toolkit/aeon/commit", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + // "pythons": ["3.9"], + + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list indicates to just test against the default (latest) + // version. + "matrix": { + "numpy": [], + "pytest": [], + "numba": [] + }, + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": "env", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "virtualenv", + // "environment_type": "mamba", + "build_cache_size": 10, + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": "benchmark_results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": "html", + + // The number of characters to retain in the commit hashes. + "hash_length": 8, + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + + "regressions_first_commits": { + "io_matlab\\.StructArr\\..*": "1a002f1" + } +} diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmarks/benchmarks/common.py b/benchmarks/benchmarks/common.py new file mode 100644 index 0000000000..e54feb3433 --- /dev/null +++ b/benchmarks/benchmarks/common.py @@ -0,0 +1,24 @@ +import os + + +class safe_import: + + def __enter__(self): + self.error = False + return self + + def __exit__(self, type_, value, traceback): + if type_ is not None: + self.error = True + suppress = not ( + os.getenv("SCIPY_ALLOW_BENCH_IMPORT_ERRORS", "1").lower() + in ("0", "false") + or not issubclass(type_, ImportError) + ) + return suppress + + +class Benchmark: + """ + Base class with sensible options + """ diff --git a/benchmarks/benchmarks/distances.py b/benchmarks/benchmarks/distances.py new file mode 100644 index 0000000000..5ba32cec67 --- /dev/null +++ b/benchmarks/benchmarks/distances.py @@ -0,0 +1,118 @@ +from abc import ABC, abstractmethod + +from .common import Benchmark, safe_import + +with safe_import(): + import aeon.distances as aeon_dists + from aeon.testing.data_generation import make_example_3d_numpy + + +class _DistanceBenchmark(Benchmark, ABC): + # params: (n_cases, n_channels, n_timepoints) + params = [ + [ + (10, 1, 10), + (100, 1, 100), + (100, 1, 500), + (10, 3, 10), + (100, 3, 100), + (100, 3, 500), + ], + ] + param_names = ["shape"] + + def setup(self, shape): + # Two independent samples so we don't measure duplicates + self.a = make_example_3d_numpy(*shape, return_y=False, random_state=1) + self.b = make_example_3d_numpy(*shape, return_y=False, random_state=2) + + def time_indv_dist(self, shape): + # single-series distance + self.distance_func(self.a[0], self.b[0]) + + def time_pairwise_dist(self, shape): + # pairwise(X) or pairwise(X, Y) + self.pairwise_func(self.a) + + def time_one_to_multiple_dist(self, shape): + self.pairwise_func(self.a[0], self.b) + + def time_multiple_to_multiple_dist(self, shape): + self.pairwise_func(self.a, self.b) + + @property + @abstractmethod + def distance_func(self): + """Return a callable: dist(Xi, Xj) -> float""" + raise NotImplementedError + + @property + @abstractmethod + def pairwise_func(self): + """Return a callable: pairwise(X[, Y]) -> ndarray""" + raise NotImplementedError + + +class _ElasticDistanceBenchmark(_DistanceBenchmark, ABC): + def time_alignment_path(self, shape): + self.alignment_func(self.a[0], self.b[0]) + + @property + @abstractmethod + def alignment_func(self): + """Return a callable: alignment_path(Xi, Xj) -> (path, cost) or similar""" + raise NotImplementedError + + +class DTW(_ElasticDistanceBenchmark): + @property + def distance_func(self): + return aeon_dists.dtw_distance + + @property + def pairwise_func(self): + return aeon_dists.dtw_pairwise_distance + + @property + def alignment_func(self): + return aeon_dists.dtw_alignment_path + + +class MSM(_ElasticDistanceBenchmark): + + @property + def distance_func(self): + return aeon_dists.msm_distance + + @property + def pairwise_func(self): + return aeon_dists.msm_pairwise_distance + + @property + def alignment_func(self): + return aeon_dists.msm_alignment_path + + +class TWE(_ElasticDistanceBenchmark): + + @property + def distance_func(self): + return aeon_dists.twe_distance + + @property + def pairwise_func(self): + return aeon_dists.twe_pairwise_distance + + @property + def alignment_func(self): + return aeon_dists.twe_alignment_path + + +class Euclidean(_DistanceBenchmark): + @property + def distance_func(self): + return aeon_dists.euclidean_distance + + @property + def pairwise_func(self): + return aeon_dists.euclidean_pairwise_distance diff --git a/benchmarks/process_global_bencmarks.py b/benchmarks/process_global_bencmarks.py new file mode 100644 index 0000000000..09d9fbebe4 --- /dev/null +++ b/benchmarks/process_global_bencmarks.py @@ -0,0 +1,49 @@ +import json + +import pandas as pd + + +def process_global_benchmarks(f): + """ + Processes the global benchmarks results into pandas DataFrame. + + Parameters + ---------- + f: {str, file-like} + Global Benchmarks output + + Returns + ------- + nfev, success_rate, mean_time + pd.DataFrame for the mean number of nfev, success_rate, mean_time + for each optimisation problem. + + Notes + ----- + Code adapted from scipy.org/scipy/benchmarks/benchmarks/process_global_benchmarks.py + """ + with open(f) as fi: + dct = json.load(fi) + + nfev = [] + nsuccess = [] + mean_time = [] + + solvers = dct[list(dct.keys())[0]].keys() + for _, results in dct.items(): + _nfev = [] + _nsuccess = [] + _mean_time = [] + for _, vals in results.items(): + _nfev.append(vals["mean_nfev"]) + _nsuccess.append(vals["nsuccess"] / vals["ntrials"] * 100) + _mean_time.append(vals["mean_time"]) + nfev.append(_nfev) + nsuccess.append(_nsuccess) + mean_time.append(_mean_time) + + nfev = pd.DataFrame(data=nfev, index=dct.keys(), columns=solvers) + nsuccess = pd.DataFrame(data=nsuccess, index=dct.keys(), columns=solvers) + mean_time = pd.DataFrame(data=mean_time, index=dct.keys(), columns=solvers) + + return nfev, nsuccess, mean_time diff --git a/pyproject.toml b/pyproject.toml index b27cc4d00f..818f9ddc2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,7 @@ dev = [ "pytest-rerunfailures", "pytest-timeout", "pytest-xdist[psutil]", + "asv" ] binder = [ "notebook", From 45b9b86f0d9aa5222492f90e3ded7864608f1106 Mon Sep 17 00:00:00 2001 From: chrisholder Date: Tue, 12 Aug 2025 16:49:43 +0100 Subject: [PATCH 02/10] added warmup --- benchmarks/benchmarks/distances.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/benchmarks/benchmarks/distances.py b/benchmarks/benchmarks/distances.py index 5ba32cec67..b11ec162b6 100644 --- a/benchmarks/benchmarks/distances.py +++ b/benchmarks/benchmarks/distances.py @@ -26,6 +26,12 @@ def setup(self, shape): self.a = make_example_3d_numpy(*shape, return_y=False, random_state=1) self.b = make_example_3d_numpy(*shape, return_y=False, random_state=2) + # Warm up + temp = make_example_3d_numpy(5, 1, 10, random_state=42, return_y=False) + for _ in range(3): + self.distance_func(temp[0], temp[0]) + self.pairwise_func(temp) + def time_indv_dist(self, shape): # single-series distance self.distance_func(self.a[0], self.b[0]) @@ -54,6 +60,14 @@ def pairwise_func(self): class _ElasticDistanceBenchmark(_DistanceBenchmark, ABC): + + def setup(self, shape): + temp = make_example_3d_numpy(5, 1, 10, random_state=42, return_y=False) + for _ in range(3): + self.alignment_func(temp[0], temp[0]) + + super().setup(shape) + def time_alignment_path(self, shape): self.alignment_func(self.a[0], self.b[0]) From 627caa7017e086fb94d64f4e9bbe292e18205169 Mon Sep 17 00:00:00 2001 From: chrisholder Date: Tue, 12 Aug 2025 19:23:48 +0100 Subject: [PATCH 03/10] cont --- benchmarks/benchmarks/distances.py | 2 +- benchmarks/html/asv.css | 161 ++ benchmarks/html/asv.js | 525 ++++++ benchmarks/html/asv_ui.js | 231 +++ benchmarks/html/graphdisplay.js | 1427 +++++++++++++++++ .../distances.Euclidean.time_indv_dist.json | 1 + ...lidean.time_multiple_to_multiple_dist.json | 1 + ...s.Euclidean.time_one_to_multiple_dist.json | 1 + ...istances.Euclidean.time_pairwise_dist.json | 1 + .../python-3.12/ram-38654705664/summary.json | 1 + .../distances.Euclidean.time_indv_dist.json | 1 + ...lidean.time_multiple_to_multiple_dist.json | 1 + ...s.Euclidean.time_one_to_multiple_dist.json | 1 + ...istances.Euclidean.time_pairwise_dist.json | 1 + benchmarks/html/index.json | 1 + benchmarks/html/info.json | 4 + benchmarks/html/jquery.flot.axislabels.js | 140 ++ benchmarks/html/regressions.css | 44 + benchmarks/html/regressions.js | 618 +++++++ benchmarks/html/regressions.json | 1 + benchmarks/html/regressions.xml | 59 + benchmarks/html/summarygrid.js | 136 ++ benchmarks/html/summarylist.css | 50 + benchmarks/html/summarylist.js | 451 ++++++ benchmarks/html/swallow.ico | Bin 0 -> 4286 bytes benchmarks/html/swallow.png | Bin 0 -> 893 bytes 26 files changed, 3858 insertions(+), 1 deletion(-) create mode 100644 benchmarks/html/asv.css create mode 100644 benchmarks/html/asv.js create mode 100644 benchmarks/html/asv_ui.js create mode 100644 benchmarks/html/graphdisplay.js create mode 100644 benchmarks/html/graphs/arch-arm64/branch-HEAD/cpu-Apple M4 Max/machine-Chriss-MBP-2/num_cpu-14/numba/numpy/os-Darwin 24.3.0/pytest/python-3.12/ram-38654705664/distances.Euclidean.time_indv_dist.json create mode 100644 benchmarks/html/graphs/arch-arm64/branch-HEAD/cpu-Apple M4 Max/machine-Chriss-MBP-2/num_cpu-14/numba/numpy/os-Darwin 24.3.0/pytest/python-3.12/ram-38654705664/distances.Euclidean.time_multiple_to_multiple_dist.json create mode 100644 benchmarks/html/graphs/arch-arm64/branch-HEAD/cpu-Apple M4 Max/machine-Chriss-MBP-2/num_cpu-14/numba/numpy/os-Darwin 24.3.0/pytest/python-3.12/ram-38654705664/distances.Euclidean.time_one_to_multiple_dist.json create mode 100644 benchmarks/html/graphs/arch-arm64/branch-HEAD/cpu-Apple M4 Max/machine-Chriss-MBP-2/num_cpu-14/numba/numpy/os-Darwin 24.3.0/pytest/python-3.12/ram-38654705664/distances.Euclidean.time_pairwise_dist.json create mode 100644 benchmarks/html/graphs/arch-arm64/branch-HEAD/cpu-Apple M4 Max/machine-Chriss-MBP-2/num_cpu-14/numba/numpy/os-Darwin 24.3.0/pytest/python-3.12/ram-38654705664/summary.json create mode 100644 benchmarks/html/graphs/summary/distances.Euclidean.time_indv_dist.json create mode 100644 benchmarks/html/graphs/summary/distances.Euclidean.time_multiple_to_multiple_dist.json create mode 100644 benchmarks/html/graphs/summary/distances.Euclidean.time_one_to_multiple_dist.json create mode 100644 benchmarks/html/graphs/summary/distances.Euclidean.time_pairwise_dist.json create mode 100644 benchmarks/html/index.json create mode 100644 benchmarks/html/info.json create mode 100644 benchmarks/html/jquery.flot.axislabels.js create mode 100644 benchmarks/html/regressions.css create mode 100644 benchmarks/html/regressions.js create mode 100644 benchmarks/html/regressions.json create mode 100644 benchmarks/html/regressions.xml create mode 100644 benchmarks/html/summarygrid.js create mode 100644 benchmarks/html/summarylist.css create mode 100644 benchmarks/html/summarylist.js create mode 100644 benchmarks/html/swallow.ico create mode 100644 benchmarks/html/swallow.png diff --git a/benchmarks/benchmarks/distances.py b/benchmarks/benchmarks/distances.py index b11ec162b6..65df4d714f 100644 --- a/benchmarks/benchmarks/distances.py +++ b/benchmarks/benchmarks/distances.py @@ -32,7 +32,7 @@ def setup(self, shape): self.distance_func(temp[0], temp[0]) self.pairwise_func(temp) - def time_indv_dist(self, shape): + def time_dist(self, shape): # single-series distance self.distance_func(self.a[0], self.b[0]) diff --git a/benchmarks/html/asv.css b/benchmarks/html/asv.css new file mode 100644 index 0000000000..d78675164e --- /dev/null +++ b/benchmarks/html/asv.css @@ -0,0 +1,161 @@ +/* Basic navigation */ + +.asv-navigation { + padding: 2px; +} + +nav ul li.active a { + height: 52px; +} + +nav li.active span.navbar-brand { + background-color: #e7e7e7; + height: 52px; +} + +nav li.active span.navbar-brand:hover { + background-color: #e7e7e7; +} + +.navbar-default .navbar-link { + color: #2458D9; +} + +.panel-body { + padding: 0; +} + +.panel { + margin-bottom: 4px; + -webkit-box-shadow: none; + box-shadow: none; + border-radius: 0; + border-top-left-radius: 3px; + border-top-right-radius: 3px; +} + +.panel-default>.panel-heading, +.panel-heading { + font-size: 12px; + font-weight:bold; + padding: 2px; + text-align: center; + border-top-left-radius: 3px; + border-top-right-radius: 3px; + background-color: #eee; +} + +.btn, +.btn-group, +.btn-group-vertical>.btn:first-child, +.btn-group-vertical>.btn:last-child:not(:first-child), +.btn-group-vertical>.btn:last-child { + border: none; + border-radius: 0px; + overflow: hidden; +} + +.btn-default:focus, .btn-default:active, .btn-default.active { + border: none; + color: #fff; + background-color: #99bfcd; +} + +#range { + font-family: monospace; + text-align: center; + background: #ffffff; +} + +.form-control { + border: none; + border-radius: 0px; + font-size: 12px; + padding: 0px; +} + +.tooltip-inner { + min-width: 100px; + max-width: 800px; + text-align: left; + white-space: pre-wrap; + font-family: monospace; +} + +/* Benchmark tree */ + +.nav-list { + font-size: 12px; + padding: 0; + padding-left: 15px; +} + +.nav-list>li { + overflow-x: hidden; +} + +.nav-list>li>a { + padding: 0; + padding-left: 5px; + color: #000; +} + +.nav-list>li>a:focus { + color: #fff; + background-color: #99bfcd; + box-shadow: inset 0 3px 5px rgba(0,0,0,.125); +} + +.nav-list>li>.nav-header { + white-space: nowrap; + font-weight: 500; + margin-bottom: 2px; +} + +.caret-right { + display: inline-block; + width: 0; + height: 0; + margin-left: 2px; + vertical-align: middle; + border-left: 4px solid; + border-bottom: 4px solid transparent; + border-top: 4px solid transparent; +} + +/* Summary page */ + +.benchmark-group > h1 { + text-align: center; +} + +.benchmark-container { + width: 300px; + height: 116px; + padding: 4px; + border-radius: 3px; +} + +.benchmark-container:hover { + background-color: #eee; +} + +.benchmark-plot { + width: 292px; + height: 88px; +} + +.benchmark-text { + font-size: 12px; + color: #000; + width: 292px; + overflow: hidden; +} + +#extra-buttons { + margin: 1em; +} + +#extra-buttons a { + border: solid 1px #ccc; +} diff --git a/benchmarks/html/asv.js b/benchmarks/html/asv.js new file mode 100644 index 0000000000..ac2356395d --- /dev/null +++ b/benchmarks/html/asv.js @@ -0,0 +1,525 @@ +'use strict'; + +$(document).ready(function() { + /* GLOBAL STATE */ + /* The index.json content as returned from the server */ + var main_timestamp = ''; + var main_json = {}; + /* Extra pages: {name: show_function} */ + var loaded_pages = {}; + /* Previous window scroll positions */ + var window_scroll_positions = {}; + /* Previous window hash location */ + var window_last_location = null; + /* Graph data cache */ + var graph_cache = {}; + var graph_cache_max_size = 5; + + var colors = [ + '#247AAD', + '#E24A33', + '#988ED5', + '#777777', + '#FBC15E', + '#8EBA42', + '#FFB5B8' + ]; + + var time_units = [ + ['ps', 'picoseconds', 0.000000000001], + ['ns', 'nanoseconds', 0.000000001], + ['μs', 'microseconds', 0.000001], + ['ms', 'milliseconds', 0.001], + ['s', 'seconds', 1], + ['m', 'minutes', 60], + ['h', 'hours', 60 * 60], + ['d', 'days', 60 * 60 * 24], + ['w', 'weeks', 60 * 60 * 24 * 7], + ['y', 'years', 60 * 60 * 24 * 7 * 52], + ['C', 'centuries', 60 * 60 * 24 * 7 * 52 * 100] + ]; + + var mem_units = [ + ['', 'bytes', 1], + ['k', 'kilobytes', 1000], + ['M', 'megabytes', 1000000], + ['G', 'gigabytes', 1000000000], + ['T', 'terabytes', 1000000000000] + ]; + + function pretty_second(x) { + for (var i = 0; i < time_units.length - 1; ++i) { + if (Math.abs(x) < time_units[i+1][2]) { + return (x / time_units[i][2]).toFixed(3) + time_units[i][0]; + } + } + + return 'inf'; + } + + function pretty_byte(x) { + for (var i = 0; i < mem_units.length - 1; ++i) { + if (Math.abs(x) < mem_units[i+1][2]) { + break; + } + } + if (i == 0) { + return x + ''; + } + return (x / mem_units[i][2]).toFixed(3) + mem_units[i][0]; + } + + function pretty_unit(x, unit) { + if (unit == "seconds") { + return pretty_second(x); + } + else if (unit == "bytes") { + return pretty_byte(x); + } + else if (unit && unit != "unit") { + return '' + x.toPrecision(3) + ' ' + unit; + } + else { + return '' + x.toPrecision(3); + } + } + + function pad_left(s, c, num) { + s = '' + s; + while (s.length < num) { + s = c + s; + } + return s; + } + + function format_date_yyyymmdd(date) { + return (pad_left(date.getFullYear(), '0', 4) + + '-' + pad_left(date.getMonth() + 1, '0', 2) + + '-' + pad_left(date.getDate(), '0', 2)); + } + + function format_date_yyyymmdd_hhmm(date) { + return (format_date_yyyymmdd(date) + ' ' + + pad_left(date.getHours(), '0', 2) + + ':' + pad_left(date.getMinutes(), '0', 2)); + } + + /* Convert a flat index to permutation to the corresponding value */ + function param_selection_from_flat_idx(params, idx) { + var selection = []; + if (idx < 0) { + idx = 0; + } + for (var k = params.length-1; k >= 0; --k) { + var j = idx % params[k].length; + selection.unshift([j]); + idx = (idx - j) / params[k].length; + } + selection.unshift([null]); + return selection; + } + + /* Convert a benchmark parameter value from their native Python + repr format to a number or a string, ready for presentation */ + function convert_benchmark_param_value(value_repr) { + var match = Number(value_repr); + if (!isNaN(match)) { + return match; + } + + /* Python str */ + match = value_repr.match(/^'(.+)'$/); + if (match) { + return match[1]; + } + + /* Python unicode */ + match = value_repr.match(/^u'(.+)'$/); + if (match) { + return match[1]; + } + + /* Python class */ + match = value_repr.match(/^$/); + if (match) { + return match[1]; + } + + return value_repr; + } + + /* Convert loaded graph data to a format flot understands, by + treating either time or one of the parameters as x-axis, + and selecting only one value of the remaining axes */ + function filter_graph_data(raw_series, x_axis, other_indices, params) { + if (params.length == 0) { + /* Simple time series */ + return raw_series; + } + + /* Compute position of data entry in the results list, + and stride corresponding to plot x-axis parameter */ + var stride = 1; + var param_stride = 0; + var param_idx = 0; + for (var k = params.length - 1; k >= 0; --k) { + if (k == x_axis - 1) { + param_stride = stride; + } + else { + param_idx += other_indices[k + 1] * stride; + } + stride *= params[k].length; + } + + if (x_axis == 0) { + /* x-axis is time axis */ + var series = new Array(raw_series.length); + for (var k = 0; k < raw_series.length; ++k) { + if (raw_series[k][1] === null) { + series[k] = [raw_series[k][0], null]; + } else { + series[k] = [raw_series[k][0], + raw_series[k][1][param_idx]]; + } + } + return series; + } + else { + /* x-axis is some parameter axis */ + var time_idx = null; + if (other_indices[0] === null) { + time_idx = raw_series.length - 1; + } + else { + /* Need to search for the correct time value */ + for (var k = 0; k < raw_series.length; ++k) { + if (raw_series[k][0] == other_indices[0]) { + time_idx = k; + break; + } + } + if (time_idx === null) { + /* No data points */ + return []; + } + } + + var x_values = params[x_axis - 1]; + var series = new Array(x_values.length); + for (var k = 0; k < x_values.length; ++k) { + if (raw_series[time_idx][1] === null) { + series[k] = [convert_benchmark_param_value(x_values[k]), + null]; + } + else { + series[k] = [convert_benchmark_param_value(x_values[k]), + raw_series[time_idx][1][param_idx]]; + } + param_idx += param_stride; + } + return series; + } + } + + function filter_graph_data_idx(raw_series, x_axis, flat_idx, params) { + var selection = param_selection_from_flat_idx(params, flat_idx); + var flat_selection = []; + $.each(selection, function(i, v) { + flat_selection.push(v[0]); + }); + return filter_graph_data(raw_series, x_axis, flat_selection, params); + } + + /* Escape special characters in graph item file names. + The implementation must match asv.util.sanitize_filename */ + function sanitize_filename(name) { + var bad_re = /[<>:"\/\\^|?*\x00-\x1f]/g; + var bad_names = ["CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", + "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", + "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", + "LPT9"]; + name = name.replace(bad_re, "_"); + if (bad_names.indexOf(name.toUpperCase()) != -1) { + name = name + "_"; + } + return name; + } + + /* Given a specific group of parameters, generate the URL to + use to load that graph. + The implementation must match asv.graph.Graph.get_file_path + */ + function graph_to_path(benchmark_name, state) { + var parts = []; + $.each(state, function(key, value) { + var part; + if (value === null) { + part = key + "-null"; + } else if (value) { + part = key + "-" + value; + } else { + part = key; + } + parts.push(sanitize_filename('' + part)); + }); + parts.sort(); + parts.splice(0, 0, "graphs"); + parts.push(sanitize_filename(benchmark_name)); + + /* Escape URI components */ + parts = $.map(parts, function (val) { return encodeURIComponent(val); }); + return parts.join('/') + ".json"; + } + + /* + Load and cache graph data (on javascript side) + */ + function load_graph_data(url, success, failure) { + var dfd = $.Deferred(); + if (graph_cache[url]) { + setTimeout(function() { + dfd.resolve(graph_cache[url]); + }, 1); + } + else { + $.ajax({ + url: url + '?timestamp=' + $.asv.main_timestamp, + dataType: "json", + cache: true + }).done(function(data) { + if (Object.keys(graph_cache).length > graph_cache_max_size) { + $.each(Object.keys(graph_cache), function (i, key) { + delete graph_cache[key]; + }); + } + graph_cache[url] = data; + dfd.resolve(data); + }).fail(function() { + dfd.reject(); + }); + } + return dfd.promise(); + } + + /* + Parse hash string, assuming format similar to standard URL + query strings + */ + function parse_hash_string(str) { + var info = {location: [''], params: {}}; + + if (str && str[0] == '#') { + str = str.slice(1); + } + if (str && str[0] == '/') { + str = str.slice(1); + } + + var match = str.match(/^([^?]*?)\?/); + if (match) { + info['location'] = decodeURIComponent(match[1]).replace(/\/+/, '/').split('/'); + var rest = str.slice(match[1].length+1); + var parts = rest.split('&'); + for (var i = 0; i < parts.length; ++i) { + var part = parts[i].split('='); + if (part.length != 2) { + continue; + } + var key = decodeURIComponent(part[0].replace(/\+/g, " ")); + var value = decodeURIComponent(part[1].replace(/\+/g, " ")); + if (value == '[none]') { + value = null; + } + if (info['params'][key] === undefined) { + info['params'][key] = [value]; + } + else { + info['params'][key].push(value); + } + } + } + else { + info['location'] = decodeURIComponent(str).replace(/\/+/, '/').split('/'); + } + return info; + } + + /* + Generate a hash string, inverse of parse_hash_string + */ + function format_hash_string(info) { + var parts = info['params']; + var str = '#' + info['location']; + + if (parts) { + str = str + '?'; + var first = true; + $.each(parts, function (key, values) { + $.each(values, function (idx, value) { + if (!first) { + str = str + '&'; + } + if (value === null) { + value = '[none]'; + } + str = str + encodeURIComponent(key) + '=' + encodeURIComponent(value); + first = false; + }); + }); + } + return str; + } + + /* + Dealing with sub-pages + */ + + function show_page(name, params) { + if (loaded_pages[name] !== undefined) { + $("#nav ul li.active").removeClass('active'); + $("#nav-li-" + name).addClass('active'); + $("#graph-display").hide(); + $("#summarygrid-display").hide(); + $("#summarylist-display").hide(); + $('#regressions-display').hide(); + $('.tooltip').remove(); + loaded_pages[name](params); + return true; + } + else { + return false; + } + } + + function hashchange() { + var info = parse_hash_string(window.location.hash); + + /* Keep track of window scroll position; makes the back-button work */ + var old_scroll_pos = window_scroll_positions[info.location.join('/')]; + window_scroll_positions[window_last_location] = $(window).scrollTop(); + window_last_location = info.location.join('/'); + + /* Redirect to correct handler */ + if (show_page(info.location, info.params)) { + /* show_page does the work */ + } + else { + /* Display benchmark page */ + info.params['benchmark'] = info.location[0]; + show_page('graphdisplay', info.params); + } + + /* Scroll back to previous position, if any */ + if (old_scroll_pos !== undefined) { + $(window).scrollTop(old_scroll_pos); + } + } + + function get_commit_hash(revision) { + var commit_hash = main_json.revision_to_hash[revision]; + if (commit_hash) { + // Return printable commit hash + commit_hash = commit_hash.slice(0, main_json.hash_length); + } + return commit_hash; + } + + function get_revision(commit_hash) { + var rev = null; + $.each(main_json.revision_to_hash, function(revision, full_commit_hash) { + if (full_commit_hash.startsWith(commit_hash)) { + rev = revision; + // break the $.each loop + return false; + } + }); + return rev; + } + + function init_index() { + /* Fetch the main index.json and then set up the page elements + based on it. */ + $.ajax({ + url: "index.json" + '?timestamp=' + $.asv.main_timestamp, + dataType: "json", + cache: true + }).done(function (index) { + main_json = index; + $.asv.main_json = index; + + /* Page title */ + var project_name = $("#project-name")[0]; + project_name.textContent = index.project; + project_name.setAttribute("href", index.project_url); + $("#project-name").textContent = index.project; + document.title = "airspeed velocity of an unladen " + index.project; + + $(window).on('hashchange', hashchange); + + $('#graph-display').hide(); + $('#regressions-display').hide(); + $('#summarygrid-display').hide(); + $('#summarylist-display').hide(); + + hashchange(); + }).fail(function () { + $.asv.ui.network_error(); + }); + } + + function init() { + /* Fetch the info.json */ + $.ajax({ + url: "info.json", + dataType: "json", + cache: false + }).done(function (info) { + main_timestamp = info['timestamp']; + $.asv.main_timestamp = main_timestamp; + init_index(); + }).fail(function () { + $.asv.ui.network_error(); + }); + } + + + /* + Set up $.asv + */ + + this.register_page = function(name, show_function) { + loaded_pages[name] = show_function; + } + this.parse_hash_string = parse_hash_string; + this.format_hash_string = format_hash_string; + + this.filter_graph_data = filter_graph_data; + this.filter_graph_data_idx = filter_graph_data_idx; + this.convert_benchmark_param_value = convert_benchmark_param_value; + this.param_selection_from_flat_idx = param_selection_from_flat_idx; + this.graph_to_path = graph_to_path; + this.load_graph_data = load_graph_data; + this.get_commit_hash = get_commit_hash; + this.get_revision = get_revision; + + this.main_timestamp = main_timestamp; /* Updated after info.json loads */ + this.main_json = main_json; /* Updated after index.json loads */ + + this.format_date_yyyymmdd = format_date_yyyymmdd; + this.format_date_yyyymmdd_hhmm = format_date_yyyymmdd_hhmm; + this.pretty_unit = pretty_unit; + this.time_units = time_units; + this.mem_units = mem_units; + + this.colors = colors; + + $.asv = this; + + + /* + Launch it + */ + + init(); +}); diff --git a/benchmarks/html/asv_ui.js b/benchmarks/html/asv_ui.js new file mode 100644 index 0000000000..af757c706e --- /dev/null +++ b/benchmarks/html/asv_ui.js @@ -0,0 +1,231 @@ +'use strict'; + +$(document).ready(function() { + function make_panel(nav, heading) { + var panel = $('
'); + nav.append(panel); + var panel_header = $( + '
' + heading + '
'); + panel.append(panel_header); + var panel_body = $('
'); + panel.append(panel_body); + return panel_body; + } + + function make_value_selector_panel(nav, heading, values, setup_callback) { + var panel_body = make_panel(nav, heading); + var vertical = false; + var buttons = $('
'); + + panel_body.append(buttons); + + $.each(values, function (idx, value) { + var button = $( + ''); + setup_callback(idx, value, button); + buttons.append(button); + }); + + return panel_body; + } + + function reflow_value_selector_panels(no_timeout) { + $('.panel').each(function (i, panel_obj) { + var panel = $(panel_obj); + panel.find('.btn-group').each(function (i, buttons_obj) { + var buttons = $(buttons_obj); + var width = 0; + + if (buttons.hasClass('reflow-done')) { + /* already processed */ + return; + } + + $.each(buttons.children(), function(idx, value) { + width += value.scrollWidth; + }); + + var max_width = panel_obj.clientWidth; + + if (width >= max_width) { + buttons.addClass("btn-group-vertical"); + buttons.css("width", "100%"); + buttons.css("max-height", "20ex"); + buttons.css("overflow-y", "auto"); + } + else { + buttons.addClass("btn-group-justified"); + } + + /* The widths can be zero if the UI is not fully layouted yet, + so mark the adjustment complete only if this is not the case */ + if (width > 0 && max_width > 0) { + buttons.addClass("reflow-done"); + } + }); + }); + + if (!no_timeout) { + /* Call again asynchronously, in case the UI was not fully layouted yet */ + setTimeout(function() { $.asv.ui.reflow_value_selector_panels(true); }, 0); + } + } + + function network_error(ajax, status, error) { + $("#error-message").text( + "Error fetching content. " + + "Perhaps web server has gone down."); + $("#error").modal('show'); + } + + function hover_graph(element, graph_url, benchmark_basename, parameter_idx, revisions) { + /* Show the summary graph as a popup */ + var plot_div = $('
'); + plot_div.css('width', '11.8em'); + plot_div.css('height', '7em'); + plot_div.css('border', '2px solid black'); + plot_div.css('background-color', 'white'); + + function update_plot() { + var markings = []; + + if (revisions) { + $.each(revisions, function(i, revs) { + var rev_a = revs[0]; + var rev_b = revs[1]; + + if (rev_a !== null) { + markings.push({ color: '#d00', lineWidth: 2, xaxis: { from: rev_a, to: rev_a }}); + markings.push({ color: "rgba(255,0,0,0.1)", xaxis: { from: rev_a, to: rev_b }}); + } + markings.push({ color: '#d00', lineWidth: 2, xaxis: { from: rev_b, to: rev_b }}); + }); + } + + $.asv.load_graph_data( + graph_url + ).done(function (data) { + var params = $.asv.main_json.benchmarks[benchmark_basename].params; + data = $.asv.filter_graph_data_idx(data, 0, parameter_idx, params); + var options = { + colors: ['#000'], + series: { + lines: { + show: true, + lineWidth: 2 + }, + shadowSize: 0 + }, + grid: { + borderWidth: 1, + margin: 0, + labelMargin: 0, + axisMargin: 0, + minBorderMargin: 0, + markings: markings, + }, + xaxis: { + ticks: [], + }, + yaxis: { + ticks: [], + min: 0 + }, + legend: { + show: false + } + }; + var plot = $.plot(plot_div, [{data: data}], options); + }).fail(function () { + // TODO: Handle failure + }); + + return plot_div; + } + + element.popover({ + placement: 'left auto', + trigger: 'hover', + html: true, + delay: 50, + content: $('
').append(plot_div) + }); + + element.on('show.bs.popover', update_plot); + } + + function hover_summary_graph(element, benchmark_basename) { + /* Show the summary graph as a popup */ + var plot_div = $('
'); + plot_div.css('width', '11.8em'); + plot_div.css('height', '7em'); + plot_div.css('border', '2px solid black'); + plot_div.css('background-color', 'white'); + + function update_plot() { + var markings = []; + + $.asv.load_graph_data( + 'graphs/summary/' + benchmark_basename + '.json' + ).done(function (data) { + var options = { + colors: $.asv.colors, + series: { + lines: { + show: true, + lineWidth: 2 + }, + shadowSize: 0 + }, + grid: { + borderWidth: 1, + margin: 0, + labelMargin: 0, + axisMargin: 0, + minBorderMargin: 0, + markings: markings, + }, + xaxis: { + ticks: [], + }, + yaxis: { + ticks: [], + min: 0 + }, + legend: { + show: false + } + }; + var plot = $.plot(plot_div, [{data: data}], options); + }).fail(function () { + // TODO: Handle failure + }); + + return plot_div; + } + + element.popover({ + placement: 'left auto', + trigger: 'hover', + html: true, + delay: 50, + content: $('
').append(plot_div) + }); + + element.on('show.bs.popover', update_plot); + } + + /* + Set up $.asv.ui + */ + + this.network_error = network_error; + this.make_panel = make_panel; + this.make_value_selector_panel = make_value_selector_panel; + this.reflow_value_selector_panels = reflow_value_selector_panels; + this.hover_graph = hover_graph; + this.hover_summary_graph = hover_summary_graph; + + $.asv.ui = this; +}); diff --git a/benchmarks/html/graphdisplay.js b/benchmarks/html/graphdisplay.js new file mode 100644 index 0000000000..ba715322b0 --- /dev/null +++ b/benchmarks/html/graphdisplay.js @@ -0,0 +1,1427 @@ +'use strict'; + +$(document).ready(function() { + /* The state of the parameters in the sidebar. Dictionary mapping + strings to arrays containing the "enabled" configurations. */ + var state = null; + /* The name of the current benchmark being displayed. */ + var current_benchmark = null; + /* An array of graphs being displayed. */ + var graphs = []; + var orig_graphs = []; + /* An array of commit revisions being displayed */ + var current_revisions = []; + /* True when log scaling is enabled. */ + var log_scale = false; + /* True when zooming in on the y-axis. */ + var zoom_y_axis = false; + /* True when log scaling is enabled. */ + var reference_scale = false; + /* True when selecting a reference point */ + var select_reference = false; + /* The reference value */ + var reference = 1.0; + /* Whether to show the legend */ + var show_legend = true; + /* Is even commit spacing being used? */ + var even_spacing = false; + var even_spacing_revisions = []; + /* Is date scale being used ? */ + var date_scale = false; + var date_to_revision = {}; + /* A little div to handle tooltip placement on the graph */ + var tooltip = null; + /* X-axis coordinate axis in the data set; always 0 for + non-parameterized tests where revision and date are the only potential x-axis */ + var x_coordinate_axis = 0; + var x_coordinate_is_category = false; + /* List of lists of value combinations to plot (apart from x-axis) + in parameterized tests. */ + var benchmark_param_selection = [[null]]; + /* Highlighted revisions */ + var highlighted_revisions = null; + /* Whether benchmark graph display was set up */ + var benchmark_graph_display_ready = false; + + + /* UTILITY FUNCTIONS */ + function arr_remove_from(a, x) { + var out = []; + $.each(a, function(i, val) { + if (x !== val) { + out.push(val); + } + }); + return out; + } + + function obj_copy(obj) { + var newobj = {}; + $.each(obj, function(key, val) { + newobj[key] = val; + }); + return newobj; + } + + function obj_length(obj) { + var i = 0; + for (var x in obj) + ++i; + return i; + } + + function obj_get_first_key(data) { + for (var prop in data) + return prop; + } + + function no_data(ajax, status, error) { + $("#error-message").text( + "No data for this combination of filters. "); + $("#error").modal('show'); + } + + function get_x_from_revision(rev) { + if (date_scale) { + return $.asv.main_json.revision_to_date[rev]; + } else { + return rev; + } + } + + function get_commit_hash(x) { + // Return the commit hash in the current graph located at position x + if (date_scale) { + x = date_to_revision[x]; + } + return $.asv.get_commit_hash(x); + } + + + function display_benchmark(bm_name, state_selection, highlight_revisions) { + setup_benchmark_graph_display(); + + $('#graph-display').show(); + $('#summarygrid-display').hide(); + $('#regressions-display').hide(); + $('.tooltip').remove(); + + if (reference_scale) { + reference_scale = false; + $('#reference').removeClass('active'); + reference = 1.0; + } + current_benchmark = bm_name; + highlighted_revisions = highlight_revisions; + $("#title").text(bm_name); + setup_benchmark_params(state_selection); + replace_graphs(); + } + + function setup_benchmark_graph_display() { + if (benchmark_graph_display_ready) { + return; + } + benchmark_graph_display_ready = true; + + /* When the window resizes, redraw the graphs */ + $(window).on('resize', function() { + update_graphs(); + }); + + var nav = $("#graphdisplay-navigation"); + + /* Make the static tooltips look correct */ + $('[data-toggle="tooltip"]').tooltip({container: 'body'}); + + /* Add insertion point for benchmark parameters */ + var state_params_nav = $("
"); + nav.append(state_params_nav); + + /* Add insertion point for benchmark parameters */ + var bench_params_nav = $("
"); + nav.append(bench_params_nav); + + /* Benchmark panel */ + var panel_body = $.asv.ui.make_panel(nav, 'benchmark'); + + var tree = $('