Cache the asv environment and fixtures, and report durations

.github/workflows/asv-benchmarking.yml

@@ -9,6 +9,9 @@ on:
 jobs:
   benchmark:
     runs-on: ubuntu-latest
+    # Without this the platform's 6h cap is the only limit, which is how a
+    # mis-skipped range quietly turns into a 453-commit build.
+    timeout-minutes: 180
     defaults:
       run:
         shell: bash -el {0}
@@ -73,15 +76,43 @@ jobs:
             cp -r uxarray-asv/results benchmarks/
           fi
 
+      - name: Cache asv's benchmark environment
+        uses: actions/cache@v6
+        with:
+          path: ${{ env.ASV_DIR }}/env
+          key: asv-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('benchmarks/asv.conf.json') }}
+          restore-keys: |
+            asv-env-${{ runner.os }}-${{ runner.arch }}-
+
+      - name: Cache the benchmark fixtures
+        uses: actions/cache@v6
+        with:
+          path: |
+            ${{ env.ASV_DIR }}/oQU*.nc
+            ${{ env.ASV_DIR }}/_io_cache
+          key: asv-fixtures-${{ runner.os }}-${{ hashFiles('benchmarks/helpers/_fixtures.py') }}
+          restore-keys: |
+            asv-fixtures-${{ runner.os }}-
+
       - name: Run benchmarks
         shell: bash -l {0}
         id: benchmark
         run: |
-          # Fill the fixture cache before asv preimports the suite, which would otherwise build it serially in the forkserver parent
-          python -m benchmarks.helpers._fixtures
+          # Fill the fixture cache before asv preimports the suite, which would
+          # otherwise build it serially in the forkserver parent. Guarded because
+          # this workflow checks out main whatever ref it was dispatched from
+          if [ -f benchmarks/helpers/_fixtures.py ]; then
+            python -m benchmarks.helpers._fixtures
+          else
+            echo "no benchmarks/helpers/_fixtures.py on this ref; nothing to prime"
+          fi
           cd benchmarks
           asv machine --machine GH-Actions --os ubuntu-latest --arch x64 --cpu "2-core unknown" --ram 7GB
-          asv run v2024.02.0..main --skip-existing --parallel || true
+          # ``--skip-existing-commits``, not ``--skip-existing``: the latter keys
+          # its skip set by (commit, env), so asv's per-commit check never
+          # matches and it builds and installs every commit in the range before
+          # skipping the benchmarks it already has.
+          asv run main^! --skip-existing-commits --parallel || true
 
       - name: Commit and push benchmark results
         run: |

.gitignore

@@ -164,3 +164,6 @@ benchmarks/env
 benchmarks/results
 benchmarks/html
 benchmarks/_io_cache
+# Generated per shard by benchmarks/helpers/_partition.py --asv-args
+benchmarks/asv.conf*.shard*.json
+benchmarks/results.shard*

benchmarks/asv.conf.hpc.json

@@ -0,0 +1,83 @@
+{
+    // Thread-scaling variant of ``asv.conf.json``, for a machine with cores to
+    // spare. Run it explicitly:
+    //
+    //     asv run --config benchmarks/asv.conf.hpc.json <range>
+    //     asv compare --config benchmarks/asv.conf.hpc.json -E conda:3.11 A B
+    //
+    // Why a second file rather than an environment variable: asv records the
+    // ``env_nobuild`` matrix into every results file it writes and folds it
+    // into the environment name, so each thread count gets its own result set
+    // and ``asv compare`` keeps them apart. A count exported in the shell is
+    // invisible to asv -- runs at 1 and 64 threads land in the *same* file for
+    // a commit and compare as though they measured the same thing. See
+    // ``benchmarks/helpers/_threads.py`` for the shell route, which is the
+    // right one for "just run this faster" and the wrong one for a study.
+    //
+    // The two mechanisms are mutually exclusive: asv layers ``env_nobuild``
+    // over the inherited environment, so ``NUMBA_NUM_THREADS`` below wins over
+    // any export.
+    //
+    // KEEP IN SYNC with asv.conf.json: ``pythons``, ``environment_type``,
+    // ``conda_channels``, ``matrix.req`` and ``build_command`` decide what gets
+    // installed. Let them drift and these numbers stop being comparable to the
+    // ones the normal config produces, silently.
+
+    "version": 1,
+    "project": "uxarray",
+    "project_url": "https://github.com/UXARRAY/uxarray",
+    "repo": "..",
+    "branches": ["main"],
+    "dvcs": "git",
+    "environment_type": "conda",
+    "conda_channels": ["conda-forge"],
+    "install_timeout": 600,
+    "launch_method": "forkserver",
+    "show_commit_url": "https://github.com/UXARRAY/uxarray/commit/",
+    "pythons": ["3.11"],
+    "benchmark_dir": ".",
+    "build_command": [
+        "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}"
+    ],
+    "build_cache_size": 4,
+
+    // A parallel kernel held at one thread takes roughly as long as the core
+    // count it would otherwise have used, so the 360s the normal config allows
+    // is not enough at the bottom of a scaling sweep.
+    "default_benchmark_timeout": 1800,
+
+    "matrix": {
+        "req": {
+            "setuptools_scm": [""],
+            "xarray": [""],
+            "netcdf4": [""],
+            "pip+pyfma": [""],
+            "tbb": [""]
+        },
+
+        // One result set per value, so the run costs its length: four values is
+        // four passes over the suite. Edit for the node -- powers of two up to
+        // its physical core count is the usual shape, and
+        // ``python -m benchmarks.helpers._threads`` reports that count.
+        // Values above the core count are allowed and simply oversubscribe.
+        // One value each: a matrix entry with several values is a separate
+        // environment per value, and asv runs the whole suite in every one of
+        // them. Only 24 of the 86 benchmarks reach a ``parallel=True`` kernel,
+        // so a four-value sweep ran the other 62 four times over to print the
+        // same number four times. ``NUMBA_NUM_THREADS`` is the one knob here;
+        // to sweep it, see ``--env`` in ``benchmarks/helpers/_partition.py``.
+        //
+        // The BLAS variables are pinned because they are what made the old
+        // sweep unreadable: numpy would take the whole node in every
+        // environment, so the ``NUMBA_NUM_THREADS=1`` column was never a
+        // single-threaded baseline and the scaling curve it implied was
+        // measuring contention. Pinned, numba's threads are the only ones.
+        "env_nobuild": {
+            "NUMBA_THREADING_LAYER": ["forksafe"],
+            "NUMBA_NUM_THREADS": ["8"],
+            "OMP_NUM_THREADS": ["1"],
+            "MKL_NUM_THREADS": ["1"],
+            "OPENBLAS_NUM_THREADS": ["1"]
+        }
+    }
+}

benchmarks/asv.conf.json

@@ -122,12 +122,27 @@
         // Belt to that brace: if TBB is ever unavailable, pick the other
         // fork-safe layer rather than quietly falling back to the one that
         // breaks. ``forksafe`` raises if no fork-safe layer exists at all.
-        "env_nobuild": {"NUMBA_THREADING_LAYER": ["forksafe"]}
+        //
+        // The BLAS variables are pinned so numpy cannot take the runner's four
+        // vCPUs out from under whatever is being measured. Unpinned it competes
+        // with numba for the same cores, which shows up as run-to-run spread in
+        // every benchmark that touches an array, not just the threaded ones.
+        // ``NUMBA_NUM_THREADS`` is deliberately left alone: numba's default is
+        // the whole machine, and with BLAS out of the way that is now an
+        // unambiguous four rather than four contended with numpy's.
+        "env_nobuild": {
+            "NUMBA_THREADING_LAYER": ["forksafe"],
+            "OMP_NUM_THREADS": ["1"],
+            "MKL_NUM_THREADS": ["1"],
+            "OPENBLAS_NUM_THREADS": ["1"]
+        }
     },
 
 
+    // Just the one command: ``python -m build`` put an sdist and a wheel in
+    // {build_dir}/dist, which asv never reads, and then this rebuilt the wheel
+    // into the directory asv actually installs from.
     "build_command": [
-        "python -m build",
         "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}"
     ],
 

benchmarks/helpers/_fixtures.py

@@ -14,11 +14,13 @@
     everything the reader produced from ``Grid.open_grid`` and
     ``Grid.open_dataset``
 
-Artifacts are keyed on both the uxarray build and the files, because an
-artifact is one version's reader output and ASV diffs commits. Likewise, there's a
-fresh read per commit. ``prime`` covers every source that is readable here,
-and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill the
-cache from a batch script instead of from inside a benchmark.
+Artifacts are keyed on the source files and nothing else, so they persist
+across runs -- and across the commits ASV diffs, which means a benchmark on the
+cached flavors measures its own subject against one fixed reader output rather
+than against a per-commit re-read. A source replaced in place misses rather than
+being served something stale. ``prime`` covers every source that is readable
+here, and there is a CLI (``python -m benchmarks.helpers._fixtures``) to fill
+the cache from a batch script instead of from inside a benchmark.
 """
 
 import hashlib

benchmarks/helpers/_machine.py

@@ -0,0 +1,166 @@
+"""Pinning the machine name asv records results under.
+
+asv keys results on a machine name and defaults it to the hostname
+(``Machine.get_defaults``), which on a hosted runner is fresh for every job --
+``runnervmgx7h7`` on one run, something else on the next. A name that never
+repeats cannot be compared across runs, and once the suite is sharded it cannot
+even be merged within one run: the file asv writes is
+``results/<machine>/<commit>-<env>.json``, so every shard has to agree on
+``<machine>`` or there is nothing for :mod:`_merge` to line up.
+
+``asv machine --machine NAME`` will not do it on its own. That command stores
+only the fields that differ from the ones it detected and then skips filling the
+rest in (``commands/machine.py``), so naming the machine is precisely what drops
+``cpu``, ``num_cpu`` and ``ram`` -- the fields that say what the timings were
+measured on, and the ones the duration report prints. Detect first with ``asv
+machine --yes``, rename after, which is what this does.
+
+Idempotent, so a job that runs it twice, or a machine file that arrives already
+pinned, is fine.
+
+Usage::
+
+    asv machine --yes
+    python -m benchmarks.helpers._machine --name gh-Linux-X64
+"""
+
+import argparse
+import json
+import os
+import platform
+import re
+import sys
+from pathlib import Path
+
+__all__ = ["pin"]
+
+_VERSION_KEY = "version"
+
+
+def default_name():
+    """A machine name that survives landing on a different node next time.
+
+    The scheduler puts you on ``derecho3`` one day and ``crhtc70`` the next, and
+    asv keys results on ``platform.uname``'s node name, so left alone it records
+    a new machine every login and the results scatter across all of them.
+
+    ``NCAR_HOST`` is the reliable answer where it is set -- it names the cluster
+    rather than the node, which is the granularity results want. Failing that,
+    the node name with its trailing digits removed, which folds ``derecho3`` and
+    ``derecho5`` together but *not* ``derecho3`` and ``crhtc70``: login and
+    compute nodes of one cluster do not share a stem. Set ``ASV_MACHINE``
+    yourself if you move between them without ``NCAR_HOST``.
+    """
+    for variable in ("ASV_MACHINE", "NCAR_HOST"):
+        value = os.environ.get(variable)
+        if value:
+            return value, variable
+    node = platform.node().split(".")[0]
+    return re.sub(r"[-_]?\d+$", "", node) or node, None
+
+
+def default_path():
+    """Where asv keeps its machine file (``MachineCollection.get_machine_file_path``)."""
+    return Path.home() / ".asv-machine.json"
+
+
+def pin(name, path=None, hostname=None, sole=False):
+    """Renames the machine file's freshly detected entry to ``name``.
+
+    Returns its details. Entries for other machines are left alone -- a runner
+    has only the one, but a login node that has recorded every compute node it
+    ever landed on should not lose them to a benchmark run.
+
+    The fresh entry is the one keyed by this host's name, since that is what
+    ``asv machine --yes`` writes (``Machine.get_defaults`` takes it from
+    ``platform.uname``). Renaming it is the whole point: several nodes of one
+    cluster should file their results under one machine, or a sharded run has
+    nothing to merge. Falls back to a lone entry whatever its name, for a runner
+    whose hostname has already been renamed away by an earlier call.
+    """
+    path = Path(path) if path is not None else default_path()
+    hostname = hostname if hostname is not None else platform.node()
+    stored = json.loads(path.read_text())
+    version = stored.pop(_VERSION_KEY, None)
+
+    if hostname in stored:
+        detected = stored.pop(hostname)
+    elif name in stored:
+        detected = stored[name]
+    elif len(stored) == 1:
+        (only,) = stored
+        detected = stored.pop(only)
+    else:
+        raise ValueError(
+            f"{path} holds {len(stored)} machines ({', '.join(sorted(stored))}), none of "
+            f"them this host ({hostname!r}) and none of them {name!r}; cannot tell which "
+            f"describes the machine this is running on. Run ``asv machine --yes`` first, "
+            f"or pass --name one of the recorded machines"
+        )
+
+    detected["machine"] = name
+    if sole:
+        stored = {}
+    stored[name] = detected
+    if version is not None:
+        stored[_VERSION_KEY] = version
+    path.write_text(json.dumps(stored, indent=4))
+    return detected
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        prog="python -m benchmarks.helpers._machine",
+        description="Rename asv's detected machine entry to a fixed name.",
+    )
+    parser.add_argument(
+        "--name",
+        default=None,
+        help="Machine name to pin to. Defaults to $ASV_MACHINE, then $NCAR_HOST, "
+        "then this host's name with trailing digits removed.",
+    )
+    parser.add_argument("--path", default=None, help="Machine file (default ~/.asv-machine.json).")
+    parser.add_argument(
+        "--hostname", default=None, help="Host whose entry to rename (default this one)."
+    )
+    parser.add_argument(
+        "--sole",
+        action="store_true",
+        help="Drop every other machine from the file. asv falls back to a lone entry "
+        "whatever the hostname, so this makes bare ``asv run``/``asv show`` work from "
+        "any node without -m. Use it where you only ever benchmark one machine.",
+    )
+    parser.add_argument(
+        "--quiet", action="store_true", help="Print only the pinned name, for capturing."
+    )
+    parser.add_argument(
+        "--print",
+        dest="print_only",
+        action="store_true",
+        help="Print the name that would be pinned and change nothing.",
+    )
+    args = parser.parse_args(argv)
+
+    name, source = (args.name, "--name") if args.name else default_name()
+    if args.print_only:
+        print(name)
+        return 0
+    detected = pin(name, args.path, args.hostname, sole=args.sole)
+    if args.quiet:
+        print(name)
+        return 0
+    print(
+        f"{name}: {detected.get('cpu', '?')} "
+        f"({detected.get('num_cpu', '?')} cpu, {detected.get('os', '?')})"
+    )
+    if source is None:
+        print(
+            f"  note: {name!r} came from this host's name. A cluster's login and compute "
+            f"nodes do not share a stem, so set ASV_MACHINE (or rely on NCAR_HOST) if you "
+            f"benchmark from both, or the results will still split in two.",
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())