task: refactor ASV benchmarks

.gitignore

@@ -4,6 +4,9 @@ build_cython
 cython_debug
 dpnp.egg-info
 
+# Airspeed Velocity (asv) benchmark environments, results and html
+benchmarks/.asv/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 

benchmarks/README.md

@@ -0,0 +1,107 @@
+# dpnp benchmarks
+
+Benchmarking dpnp using Airspeed Velocity.
+Read more about ASV [here](https://asv.readthedocs.io/en/stable/index.html).
+
+## Usage
+
+Unlike a pure-Python project, dpnp is a SYCL/DPC++ extension that requires the
+Intel oneAPI compiler and a lengthy build, so ASV does not build dpnp itself:
+`build_command` in `asv.conf.json` is empty and the benchmarks are run against
+an **existing environment** that already has dpnp installed.
+
+Create an environment
+[following these instructions](https://intelpython.github.io/dpnp/quick_start_guide.html)
+and install the benchmarking tooling into it. Either install the `benchmark`
+extra from the repo:
+
+```bash
+pip install ".[benchmark]"
+```
+
+or install `asv` directly:
+
+```bash
+conda install -c conda-forge asv
+```
+
+Then activate the environment and run the benchmarks against it. The simplest
+way is to point ASV at the currently active environment with `--python=same`:
+
+```bash
+conda activate dpnp_env
+asv run --python=same --quick HEAD^!
+```
+
+Alternatively, point ASV explicitly at an environment's python binary:
+
+```bash
+asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python
+```
+
+Compare two commits or check for regressions:
+
+```bash
+asv continuous --python=same HEAD~1 HEAD
+```
+
+For `level_zero` devices, you might see `USM Allocation` errors unless you use
+the `asv run` command with `--launch-method spawn`.
+
+By default, dpnp selects a default SYCL device. Use the `ONEAPI_DEVICE_SELECTOR`
+environment variable to target a specific device, e.g.:
+
+```bash
+ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run \
+    --launch-method spawn \
+    --python=same
+```
+
+## Benchmarks
+
+### `bench_dpbench.py` -- dpBench workloads
+
+`bench_dpbench.py` runs a set of dpnp workloads vendored from
+[dpBench](https://github.com/IntelPython/dpbench). The kernels, their data
+initialization, and the data-size presets are copied from dpBench and live in
+`benchmarks/dpbench/workloads`. Each workload is exposed as its own benchmark
+class (e.g. `BlackScholes.time_black_scholes`) and is parametrized by the
+dpBench data-size preset (`S`, `M16Gb`, `M`, `L`).
+
+Currently vendored workloads:
+
+| Workload            | Domain             |
+| ------------------- | ------------------ |
+| `black_scholes`     | Finance            |
+| `l2_norm`           | Distance Compute   |
+| `pairwise_distance` | Distance Compute   |
+| `rambo`             | Particle Physics   |
+| `gpairs`            | Astrophysics       |
+
+Host input data is generated and copied to the device exactly the way dpBench
+does, and each kernel ends with `dpnp.synchronize_array_data`, so a single call
+blocks until the device work has finished. The `time_*` methods invoke the
+workload once and let ASV wall-clock-time it (handling repeats, samples and
+statistics natively) -- the same end-to-end quantity dpBench itself measures,
+and the same plain `time_*` style used by the mkl_fft ASV benchmarks. By
+default only the small `S` preset is exercised; edit `ASV_PRESETS` in a workload
+module to benchmark larger problem sizes (which may require several GiB of
+device memory).
+
+### Other benchmark modules
+
+The remaining `bench_*.py` modules (`bench_linalg.py`, `bench_elementwise.py`,
+`bench_random.py`) are plain ASV benchmarks comparing dpnp against NumPy.
+
+## Writing new benchmarks
+
+Read ASV's guidelines for writing benchmarks
+[here](https://asv.readthedocs.io/en/stable/writing_benchmarks.html).
+
+To add another dpBench workload, copy its `<name>_dpnp.py` kernel and
+`<name>_initialize.py` initializer into a new module under
+`benchmarks/dpbench/workloads`, translate its `bench_info` TOML presets into the
+module's `PRESETS`/`ASV_PRESETS` and argument-metadata constants (see the
+existing workloads for the exact shape), and add the module to `WORKLOADS` in
+`benchmarks/dpbench/workloads/__init__.py`. `bench_dpbench.py` will generate a
+benchmark class for it automatically.

benchmarks/asv.conf.json

@@ -1,89 +1,26 @@
 {
-    // The version of the config file format.  Do not change, unless
-    // you know what you are doing.
     "version": 1,
-
-    // The name of the project being benchmarked
     "project": "dpnp",
-
-    // The project's homepage
-    "project_url": "",
-
-    // The URL or local path of the source code repository for the
-    // project being benchmarked
+    "project_url": "https://github.com/IntelPython/dpnp",
     "repo": "..",
-
-    // List of branches to benchmark. If not provided, defaults to "master"
-    // (for git) or "tip" (for mercurial).
+    "show_commit_url": "https://github.com/IntelPython/dpnp/commit/",
+    "build_command": [],
     "branches": [
         "HEAD"
     ],
-
-    // The DVCS being used.  If not set, it will be automatically
-    // determined from "repo" by looking at the protocol in the URL
-    // (if remote), or by looking for special directories, such as
-    // ".git" (if local).
     "dvcs": "git",
-
-    // The tool to use to create environments.  May be "conda",
-    // "virtualenv" or other value depending on the plugins in use.
-    // If missing or the empty string, the tool will be automatically
-    // determined by looking for tools on the PATH environment
-    // variable.
-    "environment_type": "virtualenv",
-
-    // the base URL to show a commit for the project.
-    "show_commit_url": "",
-
-    // The Pythons you'd like to test against.  If not provided, defaults
-    // to the current version of Python used to run `asv`.
-    "pythons": [
-        "3.7"
+    "environment_type": "conda",
+    "conda_channels": [
+        "https://software.repos.intel.com/python/conda/",
+        "conda-forge"
     ],
-
-    // The matrix of dependencies to test.  Each key is the name of a
-    // package (in PyPI) and the values are version numbers.  An empty
-    // list indicates to just test against the default (latest)
-    // version.
-    "matrix": {
-        "Cython": [],
-    },
-
-    // The directory (relative to the current directory) that benchmarks are
-    // stored in.  If not provided, defaults to "benchmarks"
     "benchmark_dir": "benchmarks",
-
-    // The directory (relative to the current directory) to cache the Python
-    // environments in.  If not provided, defaults to "env"
-    "env_dir": "env",
-
-    // The directory (relative to the current directory) that raw benchmark
-    // results are stored in.  If not provided, defaults to "results".
-    "results_dir": "results",
-
-    // The directory (relative to the current directory) that the html tree
-    // should be written to.  If not provided, defaults to "html".
-    "html_dir": "html",
-
-    // The number of characters to retain in the commit hashes.
-    // "hash_length": 8,
-
-    // `asv` will cache wheels of the recent builds in each
-    // environment, making them faster to install next time.  This is
-    // number of builds to keep, per environment.
-    "build_cache_size": 8,
-
-    // The commits after which the regression search in `asv publish`
-    // should start looking for regressions. Dictionary whose keys are
-    // regexps matching to benchmark names, and values corresponding to
-    // the commit (exclusive) after which to start looking for
-    // regressions.  The default is to start from the first commit
-    // with results. If the commit is `null`, regression detection is
-    // skipped for the matching benchmark.
-    //
-    // "regressions_first_commits": {
-    //    "some_benchmark": "352cdf",  // Consider regressions only after this
-    //    commit
-    //    "another_benchmark": null,   // Skip regression detection altogether
-    // }
+    "env_dir": ".asv/env",
+    "results_dir": ".asv/results",
+    "html_dir": ".asv/html",
+    "build_cache_size": 2,
+    "default_benchmark_timeout": 500,
+    "regressions_thresholds": {
+        ".*": 0.2
+    }
 }

benchmarks/benchmarks/bench_dpbench.py

@@ -0,0 +1,106 @@
+# *****************************************************************************
+# Copyright (c) 2020, Intel Corporation
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# - Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# - Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# - Neither the name of the copyright holder nor the names of its contributors
+#   may be used to endorse or promote products derived from this software
+#   without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+# THE POSSIBILITY OF SUCH DAMAGE.
+# *****************************************************************************
+
+"""ASV benchmarks for dpnp workloads vendored from dpBench.
+
+The workloads (kernels + data initialization) and their data-size presets are
+copied from dpBench (https://github.com/IntelPython/dpbench); see
+``benchmarks/benchmarks/dpbench``.
+
+Each vendored kernel ends with ``dpnp.synchronize_array_data`` on its output,
+so a single call blocks until the device work has finished. The ``time_*``
+methods below simply invoke the workload once and let ASV wall-clock-time it
+(handling repeats, samples and statistics natively) -- the same end-to-end
+quantity dpBench itself measures, and the same plain ``time_*`` style used by
+the mkl_fft ASV benchmarks.
+
+A separate benchmark class is generated for each workload -- e.g.
+``BlackScholes.time_black_scholes`` -- and parametrized by the data-size preset.
+"""
+
+import dpctl
+
+from . import benchmark_utils as bench_utils
+from .dpbench import _dpbench_runner as runner
+from .dpbench.workloads import WORKLOADS
+
+# Default-device queue, used only to query device capabilities (e.g. fp64
+# support) so unsupported-precision workloads can be skipped. This is the
+# device dpnp allocates on by default.
+DEVICE_QUEUE = dpctl.SyclQueue()
+
+
+def _camel_case(name):
+    """``black_scholes`` -> ``BlackScholes``, ``l2_norm`` -> ``L2Norm``."""
+    return "".join(part.capitalize() for part in name.split("_"))
+
+
+def _make_benchmark_class(workload):
+    """Build an ASV benchmark class for a single dpBench workload."""
+
+    class WorkloadBenchmark:
+        # The per-benchmark timeout is governed by ``default_benchmark_timeout``
+        # in ``asv.conf.json``; larger presets on a busy device can take a
+        # while.
+
+        params = list(workload.ASV_PRESETS)
+        param_names = ["preset"]
+
+        def setup(self, preset):
+            # Skip on devices that do not support the workload's precision
+            # (e.g. no fp64), mirroring the dpctl ASV benchmarks.
+            float_dtype = runner.build_types_dict(workload.PRECISION)["float"]
+            bench_utils.skip_unsupported_dtype(DEVICE_QUEUE, float_dtype)
+
+            self._runner = runner.WorkloadRunner(workload, preset)
+            self._runner.setup()
+
+        def time_workload(self, preset):
+            self._runner.run()
+
+    # Name things so ASV displays e.g. ``BlackScholes.time_black_scholes``.
+    WorkloadBenchmark.__name__ = _camel_case(workload.NAME)
+    WorkloadBenchmark.__qualname__ = WorkloadBenchmark.__name__
+
+    time_method = WorkloadBenchmark.time_workload
+    time_method.__name__ = f"time_{workload.NAME}"
+    setattr(WorkloadBenchmark, time_method.__name__, time_method)
+    del WorkloadBenchmark.time_workload
+
+    return WorkloadBenchmark
+
+
+def _generate_benchmark_classes():
+    """Create and register a benchmark class for every vendored workload."""
+    for workload in WORKLOADS:
+        cls = _make_benchmark_class(workload)
+        # Register the class at module scope so ASV can discover it.
+        globals()[cls.__name__] = cls
+
+
+_generate_benchmark_classes()

benchmarks/benchmarks/benchmark_utils.py

@@ -0,0 +1,44 @@
+# *****************************************************************************
+# Copyright (c) 2020, Intel Corporation
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# - Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# - Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# - Neither the name of the copyright holder nor the names of its contributors
+#   may be used to endorse or promote products derived from this software
+#   without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+# THE POSSIBILITY OF SUCH DAMAGE.
+# *****************************************************************************
+
+from asv_runner.benchmarks.mark import SkipNotImplemented
+
+import dpnp
+
+
+def skip_unsupported_dtype(q, dtype):
+    """Skip the benchmark if the device does not support the given dtype."""
+    dtype = dpnp.dtype(dtype)
+    device = q.sycl_device
+    if (
+        dtype in (dpnp.float64, dpnp.complex128) and not device.has_aspect_fp64
+    ) or (dtype == dpnp.float16 and not device.has_aspect_fp16):
+        raise SkipNotImplemented(
+            f"Skipping benchmark for {dtype.name} on this device"
+            + " as it is not supported."
+        )

benchmarks/benchmarks/common.py

@@ -52,10 +52,16 @@
     "int64",
     "float64",
     "complex64",
-    "longfloat",
+    # numpy.longfloat is an alias of numpy.longdouble that was removed in
+    # NumPy 2.0; use numpy.longdouble, which exists on both 1.x and 2.x.
+    "longdouble",
     "complex128",
 ]
-if "complex256" in numpy.typeDict:
+# numpy.typeDict was removed in NumPy 2.0 in favor of numpy.sctypeDict.
+_numpy_type_dict = getattr(numpy, "typeDict", None)
+if _numpy_type_dict is None:
+    _numpy_type_dict = numpy.sctypeDict
+if "complex256" in _numpy_type_dict:
     TYPES1.append("complex256")