diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 667682e..b7106fa 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -233,7 +233,7 @@ jobs: uv pip install "black==22.12.0" "coveralls>=4" \ "cytoolz==0.12.2" "dask>=2026" "isort>=5.12.0" \ "multimethod<2.0" "nbmake>=1.4.6" "numba>=0.57" \ - "numpy>=2,<3" "openmatrix==0.3.5.0" \ + "numpy>=2,<3" \ "pandera>=0.30" "pandas>=2,<3" "platformdirs>=3.2" \ "psutil>=5.9" "pyarrow>=11.0" "pydantic>=2.6,<3" "pypyr>=5.8" \ "tables>=3.9" "pytest>=7.2" "pytest-cov" "pytest-regressions" \ diff --git a/docs/walkthrough/loading-skims.ipynb b/docs/walkthrough/loading-skims.ipynb index ad8d601..97c191c 100644 --- a/docs/walkthrough/loading-skims.ipynb +++ b/docs/walkthrough/loading-skims.ipynb @@ -30,8 +30,8 @@ "from pathlib import Path\n", "from tempfile import TemporaryDirectory\n", "\n", + "import h5py\n", "import numpy as np\n", - "import openmatrix\n", "import pandas as pd\n", "import xarray as xr\n", "\n", @@ -132,7 +132,7 @@ "source": [ "## OMX: time period in matrix names\n", "\n", - "The openmatrix (OMX) standard defines a data format that stores two-dimensional matrix data. To represent a third dimension, one approach is to store a matrix for each time period and include the time period after a double underscore in the matrix name, such as `autotime__AM`." + "The Open Matrix (OMX) standard defines an HDF5 layout that stores two-dimensional matrix data. To represent a third dimension, one approach is to store a matrix for each time period and include the time period after a double underscore in the matrix name, such as `autotime__AM`." ] }, { @@ -143,15 +143,14 @@ "outputs": [], "source": [ "omx_path = data_directory / \"skims.omx\"\n", - "with openmatrix.open_file(omx_path, mode=\"w\") as omx_file:\n", + "with h5py.File(omx_path, mode=\"w\") as omx_file:\n", " for timeperiod in timeperiods:\n", - " omx_file.create_carray(\n", - " \"/data\",\n", - " f\"autotime__{timeperiod}\",\n", - " obj=source_skims.autotime.sel(timeperiod=timeperiod).values,\n", + " omx_file.create_dataset(\n", + " f\"data/autotime__{timeperiod}\",\n", + " data=source_skims.autotime.sel(timeperiod=timeperiod).values,\n", " )\n", - " omx_file.create_carray(\"/lookup\", \"taz\", obj=zones)\n", - " omx_file.root._v_attrs.SHAPE = np.array([len(zones), len(zones)])" + " omx_file.create_dataset(\"lookup/taz\", data=zones)\n", + " omx_file.attrs[\"SHAPE\"] = np.array([len(zones), len(zones)])" ] }, { diff --git a/envs/development.yml b/envs/development.yml index 31dd4ad..182e5f8 100644 --- a/envs/development.yml +++ b/envs/development.yml @@ -19,7 +19,6 @@ dependencies: - numba>=0.57 - numexpr - numpy>=1.19,<2 - - openmatrix - pandas>=1.2 - pyarrow - pytest diff --git a/envs/testing.yml b/envs/testing.yml index 5da1b97..52b5dab 100644 --- a/envs/testing.yml +++ b/envs/testing.yml @@ -21,7 +21,6 @@ dependencies: - pytest-regressions - pytest-xdist - nbmake - - openmatrix - h5py - python-blosc2 - hdf5plugin diff --git a/pyproject.toml b/pyproject.toml index e9c0913..5afb7c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,6 @@ testpaths = [ dev = [ "ipykernel>=6.31.0", "nbmake>=1.5.5", - "openmatrix>=0.3.5.0", "pytest>=8.4.2", "pytest-regressions>=2.8.3", "scipy>=1.13.1", diff --git a/sharrow/dataset.py b/sharrow/dataset.py index d68fae0..92dfb66 100755 --- a/sharrow/dataset.py +++ b/sharrow/dataset.py @@ -2,14 +2,20 @@ import ast import base64 +import concurrent.futures +import contextlib import hashlib import logging +import multiprocessing +import os import re +import secrets import time from collections.abc import Hashable, Iterable, Mapping, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any +import h5py import numpy as np import pandas as pd import pyarrow as pa @@ -23,10 +29,6 @@ from .shared_memory import si_units from .table import Table -if TYPE_CHECKING: - import openmatrix - - logger = logging.getLogger("sharrow") well_known_names = { @@ -291,20 +293,32 @@ def from_table( def _group_names(grp) -> list[str]: - """List the child node names of a pytables or h5py group.""" - try: - return list(grp._v_children) # pytables - except AttributeError: - return list(grp.keys()) # h5py + """List the child names of an HDF5 group.""" + return list(grp.keys()) def omx_file_name(omx) -> str | None: - """Resolve the on-disk filename of an OMX-like object, if possible.""" - return omx_reader.h5_filename(omx) + """Resolve the on-disk filename of an OMX HDF5 file, if possible. + + Filename discovery is deliberately structural so callers can continue to + pass handles created by optional OMX libraries. In particular, both + ``openmatrix.File`` and PyTables ``File`` expose a ``filename`` attribute; + Sharrow can reopen that path with h5py without importing either package. + """ + filename = omx_reader.h5_filename(omx) + if filename is None: + filename = getattr(omx, "filename", None) + try: + filename = os.fspath(filename) + except TypeError: + return None + if not os.path.isfile(filename): + return None + return filename def from_omx( - omx: openmatrix.File, + omx: h5py.File | str | os.PathLike, index_names=("otaz", "dtaz"), indexes="one-based", renames=None, @@ -314,13 +328,12 @@ def from_omx( Parameters ---------- - omx : openmatrix.File or larch.OMX - An OMX-format file, opened for reading. + omx : h5py.File, path-like, or filename-bearing OMX handle + An OMX-format HDF5 file, its path, or a compatible open handle such as + an ``openmatrix.File``. Filename-bearing handles are reopened through + h5py and remain owned by the caller. index_names : tuple, default ("otaz", "dtaz") - Should be a tuple of length 3, giving the names of the three - dimensions. The first two names are the native dimensions from - the open matrix file, the last is the name of the implicit - dimension that is created by parsing array names. + The names of the two matrix dimensions. indexes : str or tuple[str], optional The name of a 'lookup' in the OMX file, which will be used to populate the coordinates for the two native dimensions. Or, @@ -340,21 +353,31 @@ def from_omx( ------- Dataset """ - import h5py - - # handle larch.OMX, openmatrix.open_file, and h5py.File versions - if "lar" in type(omx).__module__: - omx_data = omx.data - omx_lookup = omx.lookup - omx_shape = omx.shape - elif isinstance(omx, h5py.File): - omx_data = omx["data"] - omx_lookup = omx["lookup"] - omx_shape = tuple(int(i) for i in omx.attrs["SHAPE"]) - else: - omx_data = omx.root["data"] - omx_lookup = omx.root["lookup"] - omx_shape = omx.shape() + if isinstance(omx, (str, os.PathLike)): + with h5py.File(omx, "r") as handle: + return from_omx( + handle, + index_names=index_names, + indexes=indexes, + renames=renames, + ) + if not isinstance(omx, h5py.File): + filename = omx_file_name(omx) + if filename is None: + raise TypeError( + "omx must be an h5py.File, path-like, or filename-bearing OMX handle" + ) + with h5py.File(filename, "r") as handle: + return from_omx( + handle, + index_names=index_names, + indexes=indexes, + renames=renames, + ) + + omx_data = omx["data"] + omx_lookup = omx["lookup"] + omx_shape = tuple(int(i) for i in omx.attrs["SHAPE"]) if renames is None: data_names = _group_names(omx_data) @@ -368,8 +391,6 @@ def from_omx( filename = omx_file_name(omx) if _is_reopenable(filename): # fast path: parallel chunk decoding via h5py - import concurrent.futures - with h5py.File(filename, "r") as f5: f5_data = f5["data"] with concurrent.futures.ThreadPoolExecutor() as pool: @@ -434,12 +455,196 @@ def _should_ignore(ignore, x): return False -def _fast_load_omx_array(filename, name): - """Load one matrix table from an OMX file, decoding chunks in parallel.""" - import h5py +def _omx_target_dtype(dtype, max_float_precision): + """Return the in-memory dtype after applying the precision limit.""" + dtype = np.dtype(dtype) + if np.issubdtype(dtype, np.floating): + max_dtype = np.dtype(f"float{max_float_precision}") + if dtype.itemsize > max_dtype.itemsize: + return max_dtype + return dtype + +def _empty_omx_3d(shape, dtype): + """Allocate a 3-D array whose individual last-axis pages are contiguous. + + OMX stores each time-period page as an independent C-contiguous 2-D HDF5 + dataset. Making the logical last axis physically outermost lets h5py read + each source page directly into the result without a matrix-sized temporary. + """ + dtype = np.dtype(dtype) + nbytes = int(np.prod(shape)) * dtype.itemsize + buffer = np.empty(nbytes, dtype=np.uint8) + strides = ( + shape[1] * dtype.itemsize, + dtype.itemsize, + shape[0] * shape[1] * dtype.itemsize, + ) + return np.ndarray(shape, dtype=dtype, buffer=buffer, strides=strides) + + +def _read_omx_dataset(dset, out=None, dtype=None): + """Read one HDF5 dataset with native h5py decompression and conversion.""" + if out is not None: + if out.flags.c_contiguous: + # HDF5 converts directly to the destination dtype when needed. + dset.read_direct(out) + else: + # h5py requires a C-contiguous destination. This fallback is used + # for ordinary C-order 3-D arrays whose last-axis pages are strided. + out[...] = dset.astype(out.dtype)[()] + return out + if dtype is None or np.dtype(dtype) == dset.dtype: + return dset[()] + return dset.astype(np.dtype(dtype))[()] + + +def _fast_load_omx_array(filename, name, dtype=None): + """Load one matrix table through h5py's native HDF5 filter pipeline.""" with h5py.File(filename, "r") as f: - return omx_reader.read_dataset(f["data"][name]) + return _read_omx_dataset(f["data"][name], dtype=dtype) + + +def _load_omx_variable(page_sources, shape, dtype): + """Load all pages of one logical OMX variable in a single Dask task.""" + result = _empty_omx_3d(shape, dtype) + open_files = {} + try: + for period, source in enumerate(page_sources): + if source is None: + result[..., period].fill(0) + continue + filename, data_name, eager_array = source + if eager_array is not None: + result[..., period] = eager_array + continue + if filename not in open_files: + open_files[filename] = h5py.File(filename, "r") + _read_omx_dataset( + open_files[filename]["data"][data_name], result[..., period] + ) + finally: + for handle in open_files.values(): + handle.close() + return result + + +def _load_omx_assignments(dataset, source, assignments): + """Load all selected matrices from one source into a prepared Dataset.""" + if isinstance(source, h5py.File): + file_context = contextlib.nullcontext(source) + else: + filename = omx_file_name(source) + if filename is None: + raise TypeError( + "OMX sources must be h5py.File, path-like, or filename-bearing handles" + ) + file_context = h5py.File(filename, "r") + bytes_loaded = 0 + with file_context as handle: + data_group = handle["data"] + for data_name, variable_name, period in assignments: + target = dataset[variable_name].data + if period is not None: + target = target[..., period] + _read_omx_dataset(data_group[data_name], target) + bytes_loaded += target.nbytes + return bytes_loaded + + +def _load_omx_shared_worker(shared_memory_key, source, assignments): + """Process worker that fills disjoint pages of a shared OMX Dataset.""" + target = xr.Dataset.shm.from_shared_memory(shared_memory_key, mode="r+") + bytes_loaded = _load_omx_assignments(target, source, assignments) + if shared_memory_key.startswith("memmap:"): + for memory_object in target.shm._shared_memory_objs_: + flush = getattr(memory_object, "flush", None) + if flush is not None: + flush() + return bytes_loaded + + +def _prepare_omx_reload_assignments(dataset, sources, time_period_sep, ignore): + """Map OMX matrices to disjoint target arrays, preserving last-file-wins.""" + assignments_by_target = {} + filenames = [] + + for source_number, source in enumerate(sources): + filename = omx_file_name(source) + filenames.append(filename) + if isinstance(source, h5py.File): + file_context = contextlib.nullcontext(source) + elif filename is not None: + file_context = h5py.File(filename, "r") + else: + raise TypeError( + "OMX sources must be h5py.File, path-like, or filename-bearing handles" + ) + + with file_context as handle: + for data_name in handle["data"]: + if _should_ignore(ignore, data_name): + logger.info("ignoring %s", data_name) + continue + + if time_period_sep in data_name: + variable_name, period_name = data_name.split(time_period_sep, 1) + if variable_name not in dataset: + logger.info( + "skipping %s because %s not in dataset", + data_name, + variable_name, + ) + continue + variable = dataset[variable_name] + if variable.ndim != 3: + raise ValueError( + f"dataset variable {variable_name} has " + f"{variable.ndim} dimensions, expected 3" + ) + period_dimension = variable.dims[-1] + try: + period = variable.get_index(period_dimension).get_loc( + period_name + ) + except KeyError: + raise KeyError( + f"time period {period_name!r} from {data_name!r} is not " + f"in dataset coordinate {period_dimension!r}" + ) from None + if not isinstance(period, (int, np.integer)): + raise ValueError( + f"time period {period_name!r} does not identify one page" + ) + period = int(period) + else: + variable_name = data_name + period = None + if variable_name not in dataset: + logger.info( + "skipping %s because it is not in dataset", data_name + ) + continue + if dataset[variable_name].ndim != 2: + raise ValueError( + f"dataset variable {variable_name} has " + f"{dataset[variable_name].ndim} dimensions, expected 2" + ) + + # Multiple source files can contain the same matrix. Matching + # from_omx_3d, the last source wins without concurrent writes. + assignments_by_target[(variable_name, period)] = ( + source_number, + data_name, + ) + + assignments = [[] for _ in sources] + for (variable_name, period), ( + source_number, + data_name, + ) in assignments_by_target.items(): + assignments[source_number].append((data_name, variable_name, period)) + return filenames, assignments def _is_reopenable(filename) -> bool: @@ -450,8 +655,6 @@ def _is_reopenable(filename) -> bool: """ if filename is None: return False - import h5py - try: with h5py.File(filename, "r"): pass @@ -461,7 +664,7 @@ def _is_reopenable(filename) -> bool: def from_omx_3d( - omx: openmatrix.File | str | Iterable[openmatrix.File | str], + omx: h5py.File | str | os.PathLike | Iterable[h5py.File | str | os.PathLike], index_names=("otaz", "dtaz", "time_period"), indexes=None, *, @@ -469,14 +672,21 @@ def from_omx_3d( time_period_sep="__", max_float_precision=32, ignore=None, + load="lazy", + task_granularity="variable", + workers=None, + memory_path=None, + shared_memory_key=None, ): """ Create a Dataset from an OMX file with an implicit third dimension. Parameters ---------- - omx : openmatrix.File or larch.OMX - An OMX-format file, opened for reading. + omx : h5py.File, path-like, filename-bearing OMX handle, or iterable + One or more OMX-format HDF5 files, paths, or compatible open handles + such as ``openmatrix.File``. Filename-bearing handles are reopened + through h5py and remain owned by the caller. index_names : tuple, default ("otaz", "dtaz", "time_period") Should be a tuple of length 3, giving the names of the three dimensions. The first two names are the native dimensions from @@ -509,73 +719,107 @@ def from_omx_3d( match the name of a variable, that variable will not be included in the loaded dataset. This is useful for excluding variables that are not needed in the current application. + load : {"lazy", "eager", "shared", "memmap"}, default "lazy" + Loading mode. ``"lazy"`` returns Dask arrays. ``"eager"`` loads into + ordinary NumPy arrays in the calling process. ``"shared"`` uses + process-parallel reads into shared memory and is the fastest mode for + multiple large OMX files. ``"memmap"`` uses the same parallel loader + with disk-backed arrays, substantially reducing resident memory at the + cost of additional storage I/O. + task_granularity : {"variable", "matrix"}, default "variable" + Dask task granularity for lazy loading. Grouping all time-period pages + of a variable minimizes graph and file-open overhead. Matrix granularity + can use less memory when only selected periods are subsequently loaded. + workers : int, optional + Number of file-level worker processes for ``"shared"`` or ``"memmap"``. + The default uses up to one worker per source file. Eager loading uses + one worker in the calling process. + memory_path : path-like, optional + New backing file to create when ``load="memmap"``. The associated + metadata is stored alongside it with a ``.meta.pkl`` suffix. Delete + both when finished with + ``result.shm.delete_shared_memory_files(result.shm.shared_memory_key)``. + shared_memory_key : str, optional + Key used to identify an explicitly shared dataset. A unique key is + generated by default. Call ``result.shm.release_shared_memory()`` when + a shared result is no longer needed. Returns ------- Dataset + Lazy Dask-backed, ordinary in-memory, shared-memory-backed, or + memory-mapped according to ``load``. """ - if not isinstance(omx, (list, tuple)): - omx = [omx] + if load is True: + load = "eager" + elif load is False: + load = "lazy" + if load not in {"lazy", "eager", "shared", "memmap"}: + raise ValueError("load must be 'lazy', 'eager', 'shared', or 'memmap'") + if task_granularity not in {"variable", "matrix"}: + raise ValueError("task_granularity must be 'variable' or 'matrix'") + if workers is not None and (not isinstance(workers, int) or workers < 1): + raise ValueError("workers must be a positive integer") + if load == "eager" and workers not in {None, 1}: + raise ValueError( + "load='eager' uses one process; use load='shared' for parallel reads" + ) + if load == "memmap" and memory_path is None: + raise ValueError("memory_path is required when load='memmap'") + if load != "memmap" and memory_path is not None: + raise ValueError("memory_path is only used when load='memmap'") + + if isinstance(omx, (h5py.File, str, os.PathLike)) or omx_file_name(omx): + omx_sources = [omx] + else: + omx_sources = list(omx) + if not omx_sources: + raise ValueError("at least one OMX file is required") use_file_handles = [] opened_file_handles = [] - for filename in omx: - if isinstance(filename, str): - import openmatrix - - h = openmatrix.open_file(filename) - opened_file_handles.append(h) - use_file_handles.append(h) - else: - use_file_handles.append(filename) - omx = use_file_handles - try: - import h5py - - # handle larch.OMX, openmatrix.open_file, and h5py.File versions - if "larch" in type(omx[0]).__module__: - omx_shape = omx[0].shape - omx_lookup = omx[0].lookup - elif isinstance(omx[0], h5py.File): - omx_shape = tuple(int(i) for i in omx[0].attrs["SHAPE"]) - omx_lookup = omx[0]["lookup"] - else: - omx_shape = omx[0].shape() - omx_lookup = omx[0].root["lookup"] - omx_data = [] - omx_data_map = {} - for n, i in enumerate(omx): - if "larch" in type(i).__module__: - omx_data.append(i.data) - elif isinstance(i, h5py.File): - omx_data.append(i["data"]) + for source in omx_sources: + if isinstance(source, (str, os.PathLike)): + h = h5py.File(source, "r") + opened_file_handles.append(h) + use_file_handles.append(h) + elif isinstance(source, h5py.File): + use_file_handles.append(source) else: - omx_data.append(i.root["data"]) - for k in _group_names(omx_data[-1]): - omx_data_map[k] = n + # Preserve compatibility with openmatrix/PyTables and similar + # optional wrappers without importing those dependencies. They + # expose the backing HDF5 path through ``filename``. + filename = omx_file_name(source) + if filename is None: + raise TypeError( + "omx entries must be h5py.File, path-like, or " + "filename-bearing OMX handles" + ) + h = h5py.File(filename, "r") + opened_file_handles.append(h) + use_file_handles.append(h) + except Exception: + for handle in opened_file_handles: + handle.close() + raise + omx_handles = use_file_handles - omx_filenames = [omx_file_name(i) for i in omx] + try: + omx_shape = tuple(int(i) for i in omx_handles[0].attrs["SHAPE"]) + omx_lookup = omx_handles[0]["lookup"] + omx_data = [handle["data"] for handle in omx_handles] + omx_data_map = {} + matrix_metadata = {} + for source_number, data_group in enumerate(omx_data): + for data_name in _group_names(data_group): + node = data_group[data_name] + omx_data_map[data_name] = source_number + matrix_metadata[data_name] = (tuple(node.shape), np.dtype(node.dtype)) + + omx_filenames = [omx_file_name(i) for i in omx_handles] omx_reopenable = [_is_reopenable(i) for i in omx_filenames] - import dask.array - - def _lazy_omx_array(k): - # Build a lazy dask array for one matrix table. When the source - # file can be independently reopened, defer to the parallel - # chunk-decoding reader; otherwise read the data eagerly, as the - # open file handle may be closed before the dask graph is computed. - n = omx_data_map[k] - filename = omx_filenames[n] - node = omx_data[n][k] - if not omx_reopenable[n]: - return dask.array.from_array(np.asarray(node[:])) - return dask.array.from_delayed( - dask.delayed(_fast_load_omx_array)(filename, k), - shape=tuple(node.shape), - dtype=node.dtype, - ) - data_names = list(omx_data_map.keys()) if ignore is not None: if isinstance(ignore, str): @@ -604,7 +848,7 @@ def _lazy_omx_array(k): r1 = ranger(n1) r2 = ranger(n2) else: - r1 = r2 = pd.Index(omx_lookup[indexes]) + r1 = r2 = pd.Index(np.asarray(omx_lookup[indexes])) if time_periods is None: raise ValueError("must give time periods explicitly") @@ -612,50 +856,240 @@ def _lazy_omx_array(k): time_periods_map = {t: n for n, t in enumerate(time_periods)} pending_3d = {} - content = {} - - for k in data_names: - if time_period_sep in k: - base_k, time_k = k.split(time_period_sep, 1) - if base_k not in pending_3d: - pending_3d[base_k] = [None] * len(time_periods) - pending_3d[base_k][time_periods_map[time_k]] = _lazy_omx_array(k) + variable_specs = {} + for data_name in data_names: + source_number = omx_data_map[data_name] + matrix_shape, matrix_dtype = matrix_metadata[data_name] + if matrix_shape != omx_shape: + raise ValueError( + f"matrix {data_name!r} has shape {matrix_shape}, expected {omx_shape}" + ) + entry = (source_number, data_name, matrix_dtype) + if time_period_sep in data_name: + base_name, period_name = data_name.split(time_period_sep, 1) + if period_name not in time_periods_map: + raise KeyError( + f"time period {period_name!r} from {data_name!r} is not in " + "time_periods" + ) + pending_3d.setdefault(base_name, [None] * len(time_periods))[ + time_periods_map[period_name] + ] = entry else: - content[k] = xr.DataArray( - _lazy_omx_array(k), - dims=index_names[:2], - coords={ - index_names[0]: r1, - index_names[1]: r2, - }, + variable_specs[data_name] = { + "pages": [entry], + "shape": omx_shape, + "dtype": _omx_target_dtype(matrix_dtype, max_float_precision), + } + for base_name, pages in pending_3d.items(): + source_dtypes = [page[2] for page in pages if page is not None] + variable_specs[base_name] = { + "pages": pages, + "shape": omx_shape + (len(time_periods),), + "dtype": _omx_target_dtype( + np.result_type(*source_dtypes), max_float_precision + ), + } + + coords = {index_names[0]: r1, index_names[1]: r2} + if pending_3d: + coords[index_names[2]] = time_periods + + # Each assignment belongs to exactly one source file. Duplicate OMX + # names retain the established last-file-wins behavior. + assignments = [[] for _ in omx_handles] + for variable_name, spec in variable_specs.items(): + is_3d = len(spec["shape"]) == 3 + for period, page in enumerate(spec["pages"]): + if page is None: + continue + source_number, data_name, _ = page + assignments[source_number].append( + (data_name, variable_name, period if is_3d else None) ) - for base_k, darrs in pending_3d.items(): - # find a prototype array - prototype = None - for i in darrs: - prototype = i - if prototype is not None: - break - if prototype is None: - raise ValueError("no prototype") - darrs_ = [ - (i if i is not None else dask.array.zeros_like(prototype)) - for i in darrs - ] - content[base_k] = xr.DataArray( - dask.array.stack(darrs_, axis=-1), - dims=index_names, - coords={ - index_names[0]: r1, - index_names[1]: r2, - index_names[2]: time_periods, - }, + + if load == "lazy": + import dask + import dask.array + + eager_arrays = {} + + def source_descriptor(page): + if page is None: + return None + source_number, data_name, _ = page + if omx_reopenable[source_number]: + return (str(omx_filenames[source_number]), data_name, None) + if data_name not in eager_arrays: + eager_arrays[data_name] = np.asarray( + omx_data[source_number][data_name][()] + ) + return (None, data_name, eager_arrays[data_name]) + + content = {} + for variable_name, spec in variable_specs.items(): + dtype = spec["dtype"] + if len(spec["shape"]) == 2: + page = spec["pages"][0] + descriptor = source_descriptor(page) + filename, data_name, eager_array = descriptor + if eager_array is not None: + array = dask.array.from_array(eager_array).astype(dtype) + else: + array = dask.array.from_delayed( + dask.delayed(_fast_load_omx_array)( + filename, data_name, dtype + ), + shape=spec["shape"], + dtype=dtype, + ) + elif task_granularity == "variable": + page_sources = [source_descriptor(i) for i in spec["pages"]] + array = dask.array.from_delayed( + dask.delayed(_load_omx_variable)( + page_sources, spec["shape"], dtype + ), + shape=spec["shape"], + dtype=dtype, + ) + else: + page_arrays = [] + for page in spec["pages"]: + descriptor = source_descriptor(page) + if descriptor is None: + page_arrays.append( + dask.array.zeros( + omx_shape, chunks=omx_shape, dtype=dtype + ) + ) + continue + filename, data_name, eager_array = descriptor + if eager_array is not None: + page_array = dask.array.from_array(eager_array).astype( + dtype + ) + else: + page_array = dask.array.from_delayed( + dask.delayed(_fast_load_omx_array)( + filename, data_name, dtype + ), + shape=omx_shape, + dtype=dtype, + ) + page_arrays.append(page_array) + array = dask.array.stack(page_arrays, axis=-1) + dims = index_names if array.ndim == 3 else index_names[:2] + content[variable_name] = (dims, array) + return xr.Dataset(content, coords=coords) + + if load == "eager": + content = {} + for variable_name, spec in variable_specs.items(): + if len(spec["shape"]) == 3: + array = _empty_omx_3d(spec["shape"], spec["dtype"]) + for period, page in enumerate(spec["pages"]): + if page is None: + array[..., period].fill(0) + else: + array = np.empty(spec["shape"], dtype=spec["dtype"]) + dims = index_names if array.ndim == 3 else index_names[:2] + content[variable_name] = (dims, array) + result = xr.Dataset(content, coords=coords) + for source, source_assignments in zip(omx_handles, assignments): + if source_assignments: + _load_omx_assignments(result, source, source_assignments) + return result + + # Shared and memory-mapped modes use a lightweight template to reserve + # one contiguous backing buffer, then independent processes fill each + # source file directly into disjoint final-array pages. + if not all(omx_reopenable[n] for n, batch in enumerate(assignments) if batch): + raise ValueError( + f"load={load!r} requires path-backed OMX files that can be reopened" + ) + import dask.array + + template_content = {} + array_order = {} + for variable_name, spec in variable_specs.items(): + dims = index_names if len(spec["shape"]) == 3 else index_names[:2] + template_content[variable_name] = ( + dims, + dask.array.empty( + spec["shape"], chunks=spec["shape"], dtype=spec["dtype"] + ), ) - for i in content: - if np.issubdtype(content[i].dtype, np.floating): - if content[i].dtype.itemsize > max_float_precision / 8: - content[i] = content[i].astype(f"float{max_float_precision}") - return xr.Dataset(content) + if len(spec["shape"]) == 3: + array_order[variable_name] = "last-axis-first" + template = xr.Dataset(template_content, coords=coords) + + if load == "memmap": + memory_path = Path(memory_path).expanduser().resolve() + metadata_path = Path(f"{memory_path}.meta.pkl") + if memory_path.exists() or metadata_path.exists(): + raise FileExistsError( + f"memory_path and metadata path must not already exist: {memory_path}" + ) + memory_path.parent.mkdir(parents=True, exist_ok=True) + key = f"memmap:{memory_path}" + else: + if shared_memory_key is not None and shared_memory_key.startswith( + "memmap:" + ): + raise ValueError("shared_memory_key must not start with 'memmap:'") + key = shared_memory_key or f"omx-{secrets.token_hex(8)}" + + result = template.shm.to_shared_memory( + key, mode="r+", load=False, array_order=array_order + ) + if hasattr(result.shm, "tasks"): + del result.shm.tasks + if hasattr(result.shm, "task_names"): + del result.shm.task_names + + active_batches = [ + (str(omx_filenames[n]), batch) + for n, batch in enumerate(assignments) + if batch + ] + worker_count = workers or max(1, min(len(active_batches), os.cpu_count() or 1)) + started = time.time() + try: + if worker_count == 1: + bytes_loaded = sum( + _load_omx_assignments(result, source, batch) + for source, batch in active_batches + ) + else: + # Spawn avoids inheriting any HDF5 state held by the caller. + mp_context = multiprocessing.get_context("spawn") + with concurrent.futures.ProcessPoolExecutor( + max_workers=worker_count, mp_context=mp_context + ) as pool: + futures = [ + pool.submit(_load_omx_shared_worker, key, source, batch) + for source, batch in active_batches + ] + bytes_loaded = sum(future.result() for future in futures) + if load == "memmap": + for memory_object in result.shm._shared_memory_objs_: + flush = getattr(memory_object, "flush", None) + if flush is not None: + flush() + logger.info( + "loaded %s from %d OMX files with %d worker(s) in %.2fs", + si_units(bytes_loaded), + len(active_batches), + worker_count, + time.time() - started, + ) + return result + except Exception: + if load == "shared": + result.shm.release_shared_memory() + else: + result.shm.delete_shared_memory_files(key) + raise finally: for h in opened_file_handles: h.close() @@ -663,27 +1097,28 @@ def _lazy_omx_array(k): def reload_from_omx_3d( dataset: xr.Dataset, - omx: Iterable[str], + omx: h5py.File | str | os.PathLike | Iterable[h5py.File | str | os.PathLike], *, time_period_sep="__", ignore=None, + workers=None, ) -> None: """ Reload the content of a dataset from OMX files. This loads the data from the OMX files into the dataset, replacing the existing data in the dataset. The dataset must have been created - by `from_omx_3d` or a similar function. Note that `from_omx_3d` will - create a dataset backed by `dask.array` objects; this function allows for - loading the data without going through dask, which may have poor performance - on some platforms. + by `from_omx_3d` or a similar function. By default, `from_omx_3d` creates + a dataset backed by `dask.array` objects; this function allows for loading + the data without going through Dask. Parameters ---------- dataset : xr.Dataset The dataset to reload into. - omx : Iterable[str] - The list of OMX file names to load from. + omx : h5py.File, path-like, filename-bearing OMX handle, or iterable + One or more OMX-format HDF5 files, paths, or compatible open handles + such as ``openmatrix.File``. time_period_sep : str, default "__" The separator used to identify time periods in the dataset. ignore : list-like, optional @@ -692,90 +1127,80 @@ def reload_from_omx_3d( match the name of a variable, that variable will not be included in the load process. This is useful for excluding variables that are not found in the target dataset. + workers : int, optional + Number of source files to load concurrently. By default, path-backed + OMX files are loaded with up to one process per source when ``dataset`` + is shared-memory-backed. Ordinary in-process datasets and sources that + cannot be reopened are loaded serially. Set to 1 to force serial I/O. """ if isinstance(ignore, str): ignore = [ignore] - - import concurrent.futures - - import h5py - - bytes_loaded = 0 - t0 = time.time() - - def _load_into(raw, dset, executor): - """Fill the array `raw` from the h5py or pytables dataset `dset`.""" - if isinstance(dset, h5py.Dataset): - if raw.dtype == dset.dtype: - omx_reader.read_dataset(dset, out=raw, executor=executor) - else: - raw[:, :] = omx_reader.read_dataset(dset, executor=executor) - else: - raw[:, :] = dset[:, :] - - def _load_one(dset, data_name, filter_note, executor): - nonlocal bytes_loaded - t1 = time.time() - if time_period_sep in data_name: - data_name_x, data_name_t = data_name.split(time_period_sep, 1) - if data_name_x not in dataset: - logger.info( - f"skipping {data_name} because {data_name_x} not in dataset" - ) - return - if len(dataset[data_name_x].dims) != 3: - raise ValueError( - f"dataset variable {data_name_x} has " - f"{len(dataset[data_name_x].dims)} dimensions, expected 3" - ) - raw = dataset[data_name_x].sel(time_period=data_name_t).data - else: - if len(dataset[data_name].dims) != 2: - raise ValueError( - f"dataset variable {data_name} has " - f"{len(dataset[data_name].dims)} dimensions, expected 2" - ) - raw = dataset[data_name].data - _load_into(raw, dset, executor) - bytes_loaded += raw.nbytes - logger.debug( - f"loaded {data_name} ({filter_note}) to dataset " - f"in {time.time() - t1:.2f}s, {si_units(bytes_loaded)}" + if isinstance(omx, (h5py.File, str, os.PathLike)) or omx_file_name(omx): + sources = [omx] + else: + sources = list(omx) + if workers is not None and (not isinstance(workers, int) or workers < 1): + raise ValueError("workers must be a positive integer") + + filenames, assignments = _prepare_omx_reload_assignments( + dataset, sources, time_period_sep, ignore + ) + active_batches = [ + (source_number, batch) + for source_number, batch in enumerate(assignments) + if batch + ] + if not active_batches: + logger.info("no OMX matrices selected for reload") + return + + parallel_capable = dataset.shm.is_shared_memory and all( + _is_reopenable(filenames[source_number]) for source_number, _ in active_batches + ) + if workers is None: + worker_count = ( + max(1, min(len(active_batches), os.cpu_count() or 1)) + if parallel_capable + else 1 ) + else: + worker_count = min(workers, len(active_batches)) + if worker_count > 1 and not parallel_capable: + raise ValueError( + "parallel OMX reload requires a shared-memory-backed dataset " + "and path-backed source files" + ) - opened_file_handles = [] - try: - with concurrent.futures.ThreadPoolExecutor() as pool: - for source in omx: - filename = omx_file_name(source) - if filename is not None: - # fast path: parallel chunk decoding via h5py - logger.info(f"loading into dataset from {filename}") - f5 = h5py.File(filename, "r") - opened_file_handles.append(f5) - data_group = f5["data"] - for data_name in data_group.keys(): - if _should_ignore(ignore, data_name): - logger.info(f"ignoring {data_name}") - continue - dset = data_group[data_name] - filter_note = f"{dset.compression}/{dset.compression_opts}" - _load_one(dset, data_name, filter_note, pool) - else: - # source is an open file handle with no resolvable - # on-disk filename; read through the handle directly - f = source - for data_name in f.root.data._v_children: - if _should_ignore(ignore, data_name): - logger.info(f"ignoring {data_name}") - continue - filters = f.root.data[data_name].filters - filter_note = f"{filters.complib}/{filters.complevel}" - _load_one(f.root.data[data_name], data_name, filter_note, pool) - logger.info(f"loading to dataset complete in {time.time() - t0:.2f}s") - finally: - for h in opened_file_handles: - h.close() + started = time.time() + if worker_count == 1: + bytes_loaded = sum( + _load_omx_assignments(dataset, sources[source_number], batch) + for source_number, batch in active_batches + ) + else: + shared_memory_key = dataset.shm.shared_memory_key + mp_context = multiprocessing.get_context("spawn") + with concurrent.futures.ProcessPoolExecutor( + max_workers=worker_count, mp_context=mp_context + ) as pool: + futures = [ + pool.submit( + _load_omx_shared_worker, + shared_memory_key, + str(filenames[source_number]), + batch, + ) + for source_number, batch in active_batches + ] + bytes_loaded = sum(future.result() for future in futures) + + logger.info( + "reloaded %s from %d OMX files with %d worker(s) in %.2fs", + si_units(bytes_loaded), + len(active_batches), + worker_count, + time.time() - started, + ) def _parquet_layout(labels_0, labels_1): diff --git a/sharrow/example_data.py b/sharrow/example_data.py index 10ec665..b212ccb 100644 --- a/sharrow/example_data.py +++ b/sharrow/example_data.py @@ -3,6 +3,7 @@ from importlib.resources import as_file, files from pathlib import Path +import h5py import numpy as np import pandas as pd @@ -28,13 +29,10 @@ def get_skims_filename() -> str: def get_skims_omx(): - import openmatrix - from . import dataset with get_example_data_path("skims.omx") as filename: - skims = None - with openmatrix.open_file(str(filename)) as f: + with h5py.File(filename, "r") as f: skims = dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), diff --git a/sharrow/omx.py b/sharrow/omx.py index 1340779..8665c15 100644 --- a/sharrow/omx.py +++ b/sharrow/omx.py @@ -1,59 +1,80 @@ import logging -import os +from pathlib import Path + +import h5py logger = logging.getLogger("sharrow.omx") +def _initialize_omx_file(source: h5py.File, target: h5py.File) -> None: + """Copy OMX metadata and ensure the standard HDF5 groups exist.""" + for name, value in source.attrs.items(): + target.attrs[name] = value + for group_name in ("data", "lookup"): + source_group = source[group_name] + target_group = target.require_group(group_name) + for name, value in source_group.attrs.items(): + target_group.attrs[name] = value + + +def _copy_dataset( + source_group: h5py.Group, target_group: h5py.Group, name: str +) -> None: + """Copy one HDF5 dataset, replacing a same-named target if present.""" + if name in target_group: + del target_group[name] + source_group.copy(name, target_group, name=name) + + def split_omx(source_file, dest_directory, global_lookups=False, n_chunks=None): - """ - Split an OMX file into separate files for each element. + """Split the matrices in an OMX file across smaller OMX files. Parameters ---------- - source_file : str - dest_directory : str - global_lookups : bool - + source_file : str or path-like + OMX-format HDF5 file to split. + dest_directory : str or path-like + Directory in which to write the split files. + global_lookups : bool, default False + If true, copy all lookups into every matrix output. Otherwise, write + each lookup to its own OMX file. + n_chunks : int, optional + Number of output matrix files. By default, write one file per matrix. """ - try: - from larch import OMX - except ImportError: - raise ImportError("larch is required to split OMX files") from None - - source = OMX(source_file, mode="r") - os.makedirs(dest_directory, exist_ok=True) - - if n_chunks is not None: - general_name = os.path.splitext(source_file)[0] - chunkfiles = [f"{general_name}-chunk{n}.omx" for n in range(n_chunks)] - else: - chunkfiles = [f"{k}.omx" for k in list(source.data._v_children)] - - n = 0 - for k in list(source.data._v_children): - newfile = os.path.join( - dest_directory, - chunkfiles[n], - ) - logger.info(f"writing {k} to {newfile}") - b = OMX(newfile, mode="a") - b.add_matrix(k, source.data[k], overwrite=True) + if n_chunks is not None and n_chunks < 1: + raise ValueError("n_chunks must be a positive integer") + + source_path = Path(source_file) + destination = Path(dest_directory) + destination.mkdir(parents=True, exist_ok=True) + + with h5py.File(source_path, "r") as source: + matrix_names = list(source["data"].keys()) + if n_chunks is not None: + chunk_names = [ + f"{source_path.stem}-chunk{number}.omx" for number in range(n_chunks) + ] + else: + chunk_names = [f"{name}.omx" for name in matrix_names] + + output_paths = [] + for number, matrix_name in enumerate(matrix_names): + output_path = destination / chunk_names[number % len(chunk_names)] + output_paths.append(output_path) + logger.info(f"writing {matrix_name} to {output_path}") + with h5py.File(output_path, "a") as target: + _initialize_omx_file(source, target) + _copy_dataset(source["data"], target["data"], matrix_name) + if global_lookups: - # todo only write once per file - for j in list(source.lookup._v_children): - b.add_lookup(j, source.lookup[j], overwrite=True) - b.close() - n += 1 - if n >= len(chunkfiles): - n = 0 - if not global_lookups: - for k in list(source.lookup._v_children): - newfile = os.path.join( - dest_directory, - f"_{k}.omx", - ) - logger.info(f"writing {newfile}") - b = OMX(newfile, mode="w") - b.shape = source.shape - b.add_lookup(k, source.lookup[k]) - b.close() + for output_path in dict.fromkeys(output_paths): + with h5py.File(output_path, "a") as target: + for lookup_name in source["lookup"]: + _copy_dataset(source["lookup"], target["lookup"], lookup_name) + else: + for lookup_name in source["lookup"]: + output_path = destination / f"_{lookup_name}.omx" + logger.info(f"writing {output_path}") + with h5py.File(output_path, "w") as target: + _initialize_omx_file(source, target) + _copy_dataset(source["lookup"], target["lookup"], lookup_name) diff --git a/sharrow/omx_reader.py b/sharrow/omx_reader.py index e9321b3..a455554 100644 --- a/sharrow/omx_reader.py +++ b/sharrow/omx_reader.py @@ -50,14 +50,13 @@ ) -def h5_filename(omx) -> str | None: - """Resolve the on-disk filename of an OMX-like object. +def h5_filename(omx: h5py.File | str | os.PathLike) -> str | None: + """Resolve the on-disk filename of an OMX HDF5 file. Parameters ---------- - omx : str, os.PathLike, h5py.File, tables.File, or larch.OMX - An OMX file reference: a path, an h5py file, a pytables file (which - includes ``openmatrix.File``), or a larch OMX object. + omx : str, os.PathLike, or h5py.File + An OMX file path or open HDF5 file. Returns ------- @@ -66,8 +65,8 @@ def h5_filename(omx) -> str | None: """ if isinstance(omx, (str, os.PathLike)): return os.fspath(omx) - # h5py.File, tables.File (and openmatrix.File), and larch.OMX all expose - # the underlying file path as a `filename` attribute. + if not isinstance(omx, h5py.File): + return None filename = getattr(omx, "filename", None) if isinstance(filename, (str, os.PathLike)): filename = os.fspath(filename) diff --git a/sharrow/shared_memory.py b/sharrow/shared_memory.py index d93b38f..3a4c739 100644 --- a/sharrow/shared_memory.py +++ b/sharrow/shared_memory.py @@ -246,7 +246,25 @@ def __repr__(self): def release_shared_memory(self): """Release shared memory allocated to this Dataset.""" - release_shared_memory(self._shared_memory_key_) + key = self._shared_memory_key_ + if key and key.startswith("memmap:"): + # Memmaps are intentionally not kept in the process-global shared + # memory registry. Release the Dataset's own buffer and mapping so + # Windows can delete or replace the backing file immediately. + buffer = getattr(self, "_buffer", None) + if buffer is not None: + buffer.release() + del self._buffer + for memory_object in self._shared_memory_objs_: + if isinstance(memory_object, np.memmap): + memory_object.flush() + mmap = getattr(memory_object, "_mmap", None) + if mmap is not None and not mmap.closed: + mmap.close() + self._shared_memory_objs_.clear() + self._shared_memory_owned_ = False + else: + release_shared_memory(key) @staticmethod def delete_shared_memory_files(key): @@ -260,6 +278,7 @@ def to_shared_memory( dask_scheduler="threads", pre_init=False, load=True, + array_order=None, ): """ Load this Dataset into shared memory. @@ -291,6 +310,11 @@ def to_shared_memory( the `shm.tasks` attribute of the resulting Dataset object, but do not necessarily need to be run if data can be loaded using alternative methods (e.g. `sharrow.dataset.reload_from_omx_3d`). + array_order : mapping, optional + Optional storage order overrides keyed by variable name. The + ``"last-axis-first"`` order stores each page along the last axis + contiguously. This is useful when a three-dimensional array is + populated from a collection of two-dimensional source arrays. Returns ------- @@ -336,18 +360,33 @@ def emit(k, a, is_coord): a_nbytes = a.data.nbytes else: logger.debug(f"preparing dense array {a.name}") - wrappers.append( - { - "dims": a.dims, - "name": a.name, - "attrs": a.attrs, - "dtype": a.dtype, - "shape": a.shape, - "coord": is_coord, - "nbytes": a.nbytes, - "position": position, - } - ) + wrapper = { + "dims": a.dims, + "name": a.name, + "attrs": a.attrs, + "dtype": a.dtype, + "shape": a.shape, + "coord": is_coord, + "nbytes": a.nbytes, + "position": position, + } + order = None if array_order is None else array_order.get(k) + if order == "last-axis-first": + if a.ndim < 2: + raise ValueError( + "last-axis-first storage requires at least two dimensions" + ) + # The logical last axis is physically outermost, making + # each ``[..., page]`` view C contiguous for direct I/O. + itemsize = a.dtype.itemsize + strides = [itemsize] + for size in reversed(a.shape[1:-1]): + strides.insert(0, strides[0] * size) + strides.append(int(np.prod(a.shape[:-1])) * itemsize) + wrapper["strides"] = tuple(strides) + elif order is not None: + raise ValueError(f"unknown shared-memory array order {order!r}") + wrappers.append(wrapper) a_nbytes = a.nbytes sizes.append(a_nbytes) @@ -372,8 +411,9 @@ def emit(k, a, is_coord): if pre_init: logger.debug("pre-initializing shared memory buffer") - # gross init with all zeros - buffer[:] = b"\0" * len(buffer) + # Fill in place. Constructing a same-sized bytes object can + # temporarily double memory use for multi-gigabyte skim datasets. + np.ndarray(len(buffer), dtype=np.uint8, buffer=buffer).fill(0) tasks = [] task_names = [] @@ -412,7 +452,10 @@ def emit(k, a, is_coord): else: logger.debug(f"preparing load task: {_name} ({si_units(_size)})") mem_arr = np.ndarray( - shape=a.shape, dtype=a.dtype, buffer=buffer[_pos : _pos + _size] + shape=a.shape, + dtype=a.dtype, + buffer=buffer[_pos : _pos + _size], + strides=w.get("strides"), ) if isinstance(a, xr.DataArray) and isinstance(a.data, da.Array): tasks.append(da.store(a.data, mem_arr, lock=False, compute=False)) @@ -500,7 +543,7 @@ def from_shared_memory(cls, key, own_data=False, mode="r+"): # for memmap, list is loaded from pickle, not shared ram pass - if own_data and own_data is not True: + if own_data is not None and not isinstance(own_data, (bool, np.bool_)): mem = own_data own_data = True else: @@ -523,6 +566,7 @@ def from_shared_memory(cls, key, own_data=False, mode="r+"): coord = t.pop("coord", False) # noqa: F841 position = t.pop("position") nbytes = t.pop("nbytes") + strides = t.pop("strides", None) is_sparse = t.pop("sparse", False) if is_sparse: if sparse is None: @@ -565,7 +609,10 @@ def from_shared_memory(cls, key, own_data=False, mode="r+"): ) else: mem_arr = np.ndarray( - shape, dtype=dtype, buffer=buffer[position : position + nbytes] + shape, + dtype=dtype, + buffer=buffer[position : position + nbytes], + strides=strides, ) content[name] = DataArray(mem_arr, **t) diff --git a/sharrow/tests/test_datasets.py b/sharrow/tests/test_datasets.py index a48b3e2..ee824b0 100644 --- a/sharrow/tests/test_datasets.py +++ b/sharrow/tests/test_datasets.py @@ -2,8 +2,8 @@ import tempfile from pathlib import Path +import h5py import numpy as np -import openmatrix import pandas as pd import pytest import xarray as xr @@ -16,15 +16,12 @@ def test_dataset_construct_with_zoneids(): tempdir = tempfile.TemporaryDirectory() t = Path(tempdir.name) - with openmatrix.open_file(t.joinpath("dummy5.omx"), mode="w") as out: - out.create_carray("/data", "Eye", obj=np.eye(5, dtype=np.float32)) - out.create_carray("/lookup", "Zone", obj=np.asarray([11, 22, 33, 44, 55])) - shp = np.empty(2, dtype=int) - shp[0] = 5 - shp[1] = 5 - out.root._v_attrs.SHAPE = shp + with h5py.File(t.joinpath("dummy5.omx"), mode="w") as out: + out.create_dataset("data/Eye", data=np.eye(5, dtype=np.float32)) + out.create_dataset("lookup/Zone", data=np.asarray([11, 22, 33, 44, 55])) + out.attrs["SHAPE"] = np.asarray([5, 5], dtype=int) - with openmatrix.open_file(t.joinpath("dummy5.omx"), mode="r") as back: + with h5py.File(t.joinpath("dummy5.omx"), mode="r") as back: ds = sh.dataset.from_omx(back, indexes="Zone") assert sorted(ds.coords) == ["dtaz", "otaz"] @@ -32,11 +29,11 @@ def test_dataset_construct_with_zoneids(): assert sorted(ds.variables) == ["Eye", "dtaz", "otaz"] assert ds["Eye"].data == approx(np.eye(5, dtype=np.float32)) - with openmatrix.open_file(t.joinpath("dummy5.omx"), mode="r") as back: + with h5py.File(t.joinpath("dummy5.omx"), mode="r") as back: ds0 = sh.dataset.from_omx(back, indexes="zero-based") assert ds0.coords["otaz"].values == approx(np.asarray([0, 1, 2, 3, 4])) - with openmatrix.open_file(t.joinpath("dummy5.omx"), mode="r") as back: + with h5py.File(t.joinpath("dummy5.omx"), mode="r") as back: ds1 = sh.dataset.from_omx(back, indexes="one-based") assert ds1.coords["otaz"].values == approx(np.asarray([1, 2, 3, 4, 5])) @@ -72,7 +69,7 @@ def income_cat(i): def test_load_with_ignore(): filename = sh.example_data.get_skims_filename() - with openmatrix.open_file(filename) as f: + with h5py.File(filename) as f: skims = sh.dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), @@ -83,7 +80,7 @@ def test_load_with_ignore(): ) assert "DRV_COM_WLK_FAR" in skims.variables - with openmatrix.open_file(filename) as f: + with h5py.File(filename) as f: skims1 = sh.dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), @@ -95,7 +92,7 @@ def test_load_with_ignore(): ) assert "DRV_COM_WLK_FAR" not in skims1.variables - with openmatrix.open_file(filename) as f: + with h5py.File(filename) as f: skims2 = sh.dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), @@ -121,7 +118,7 @@ def test_deferred_load_to_shared_memory(): from sharrow.example_data import get_skims_filename skims_filename = get_skims_filename() - with openmatrix.open_file(skims_filename) as f: + with h5py.File(skims_filename) as f: d0 = sh.dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), @@ -315,27 +312,23 @@ def test_from_parquet_3d_ignore(): assert "DIST" in skims.variables -def _write_compressed_omx(path, matrices, complib="zlib", complevel=7, shuffle=True): +def _write_compressed_omx(path, matrices, compression="gzip", compression_opts=7): """Write an OMX file with compressed, oddly-chunked matrix tables.""" - import tables - - filters = tables.Filters(complevel=complevel, complib=complib, shuffle=shuffle) n1, n2 = next(iter(matrices.values())).shape - with openmatrix.open_file(path, mode="w") as out: + with h5py.File(path, mode="w") as out: for name, arr in matrices.items(): - out.create_carray( - "/data", - name, - obj=arr, - filters=filters, + out.create_dataset( + f"data/{name}", + data=arr, + compression=compression, + compression_opts=compression_opts, + shuffle=True, # chunk shape that does not evenly divide the array, # to exercise edge-chunk handling - chunkshape=(7, 7), + chunks=(7, 7), ) - out.create_carray("/lookup", "taz", obj=np.arange(11, 11 + n1)) - shp = np.empty(2, dtype=int) - shp[0], shp[1] = n1, n2 - out.root._v_attrs.SHAPE = shp + out.create_dataset("lookup/taz", data=np.arange(11, 11 + n1)) + out.attrs["SHAPE"] = np.asarray([n1, n2], dtype=int) def _random_matrices(n=25, seed=42): @@ -353,7 +346,7 @@ def test_from_omx_compressed_zlib(): with tempfile.TemporaryDirectory() as tempdir: f = Path(tempdir).joinpath("skims.omx") _write_compressed_omx(f, matrices) - with openmatrix.open_file(f, mode="r") as back: + with h5py.File(f, mode="r") as back: ds = sh.dataset.from_omx(back, indexes="taz") ds_renamed = sh.dataset.from_omx( back, indexes="taz", renames={"distance": "DIST"} @@ -367,20 +360,55 @@ def test_from_omx_compressed_zlib(): assert sorted(ds_limited.data_vars) == ["COUNTS"] -def test_from_omx_compressed_h5py_handle(): - import h5py - +def test_from_omx_compressed_path(): matrices = _random_matrices() with tempfile.TemporaryDirectory() as tempdir: f = Path(tempdir).joinpath("skims.omx") _write_compressed_omx(f, matrices) - with h5py.File(f, mode="r") as back: - ds = sh.dataset.from_omx(back, indexes="taz") + ds = sh.dataset.from_omx(f, indexes="taz") for name, arr in matrices.items(): np.testing.assert_array_equal(ds[name].values, arr) assert ds.coords["otaz"].values == approx(np.arange(11, 36)) +def test_from_omx_filename_bearing_legacy_handle(): + """Legacy OMX handles are reopened by filename without being imported.""" + + class LegacyOMXHandle: + """Minimal protocol exposed by openmatrix.File and PyTables File.""" + + def __init__(self, filename): + self.filename = filename + self.close_called = False + + def close(self): + self.close_called = True + + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + f = Path(tempdir).joinpath("skims.omx") + _write_compressed_omx(f, matrices) + legacy_handle = LegacyOMXHandle(f) + + expected_2d = sh.dataset.from_omx(f, indexes="taz") + actual_2d = sh.dataset.from_omx(legacy_handle, indexes="taz") + xr.testing.assert_equal(actual_2d, expected_2d) + + expected_3d = sh.dataset.from_omx_3d(f, time_periods=["AM", "PM"], load="eager") + actual_3d = sh.dataset.from_omx_3d( + [legacy_handle], time_periods=["AM", "PM"] + ).compute() + xr.testing.assert_equal(actual_3d, expected_3d) + + reloaded = xr.zeros_like(expected_3d) + sh.dataset.reload_from_omx_3d(reloaded, legacy_handle) + xr.testing.assert_equal(reloaded, expected_3d) + + # Sharrow owns only the temporary h5py handle it creates. The caller's + # compatibility handle remains open and under caller control. + assert not legacy_handle.close_called + + def test_from_omx_3d_compressed_zlib(): matrices = _random_matrices() with tempfile.TemporaryDirectory() as tempdir: @@ -402,7 +430,7 @@ def test_from_omx_3d_compressed_zlib(): assert skims.coords["otaz"].values == approx(np.arange(11, 36)) # also via an already-open file handle - with openmatrix.open_file(f, mode="r") as back: + with h5py.File(f, mode="r") as back: skims2 = sh.dataset.from_omx_3d( back, time_periods=["AM", "PM"], @@ -412,6 +440,120 @@ def test_from_omx_3d_compressed_zlib(): xr.testing.assert_equal(skims, skims2.compute()) +def test_from_omx_3d_loading_modes(): + """Lazy batching and direct eager loading produce identical skim arrays.""" + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + f = Path(tempdir).joinpath("skims.omx") + _write_compressed_omx(f, matrices) + lazy_variable = sh.dataset.from_omx_3d( + f, + time_periods=["EA", "AM", "PM"], + task_granularity="variable", + ) + lazy_matrix = sh.dataset.from_omx_3d( + f, + time_periods=["EA", "AM", "PM"], + task_granularity="matrix", + ) + eager = sh.dataset.from_omx_3d(f, time_periods=["EA", "AM", "PM"], load="eager") + + # One task per logical variable substantially reduces scheduler work. + assert len(lazy_variable.__dask_graph__()) < len(lazy_matrix.__dask_graph__()) + xr.testing.assert_equal(lazy_variable.compute(), eager) + xr.testing.assert_equal(lazy_matrix.compute(), eager) + assert (eager["TIME"].sel(time_period="EA").values == 0).all() + assert eager["TIME"].data[..., 0].flags.c_contiguous + assert eager["DIST"].dtype == np.float32 + + +def test_from_omx_3d_shared_parallel(): + """Separate OMX files can load concurrently into final shared pages.""" + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + first = Path(tempdir).joinpath("first.omx") + second = Path(tempdir).joinpath("second.omx") + _write_compressed_omx( + first, {name: data for name, data in matrices.items() if name != "TIME__PM"} + ) + _write_compressed_omx(second, {"TIME__PM": matrices["TIME__PM"]}) + expected = sh.dataset.from_omx_3d( + [first, second], time_periods=["EA", "AM", "PM"], load="eager" + ) + token = "parallel-skims-" + secrets.token_hex(5) + shared = sh.dataset.from_omx_3d( + [first, second], + time_periods=["EA", "AM", "PM"], + load="shared", + workers=2, + shared_memory_key=token, + ) + try: + xr.testing.assert_equal(shared, expected) + assert shared.shm.is_shared_memory + assert shared["TIME"].data[..., 1].flags.c_contiguous + finally: + shared.shm.release_shared_memory() + + +def test_from_omx_3d_memmap_low_memory(): + """The low-memory mode loads directly into a new disk-backed array.""" + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + f = Path(tempdir).joinpath("skims.omx") + backing = Path(tempdir).joinpath("skims-memory.dat") + _write_compressed_omx(f, matrices) + expected = sh.dataset.from_omx_3d( + f, time_periods=["EA", "AM", "PM"], load="eager" + ) + mapped = sh.dataset.from_omx_3d( + f, + time_periods=["EA", "AM", "PM"], + load="memmap", + memory_path=backing, + workers=2, + ) + key = mapped.shm.shared_memory_key + try: + xr.testing.assert_equal(mapped, expected) + assert backing.exists() + assert Path(f"{backing}.meta.pkl").exists() + assert isinstance(mapped.shm._shared_memory_objs_[-1], np.memmap) + + with pytest.raises(FileExistsError): + sh.dataset.from_omx_3d( + f, + time_periods=["EA", "AM", "PM"], + load="memmap", + memory_path=backing, + ) + finally: + mapped.shm.release_shared_memory() + mapped.shm.delete_shared_memory_files(key) + + assert not backing.exists() + assert not Path(f"{backing}.meta.pkl").exists() + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"load": "invalid"}, "load must be"), + ({"task_granularity": "invalid"}, "task_granularity"), + ({"load": "shared", "workers": 0}, "positive integer"), + ({"load": "memmap"}, "memory_path is required"), + ({"load": "eager", "workers": 2}, "load='shared'"), + ], +) +def test_from_omx_3d_loading_mode_validation(kwargs, message): + with pytest.raises(ValueError, match=message): + sh.dataset.from_omx_3d( + sh.example_data.get_skims_filename(), + time_periods=["EA", "AM", "MD", "PM", "EV"], + **kwargs, + ) + + def test_reload_from_omx_3d_compressed(): matrices = _random_matrices() with tempfile.TemporaryDirectory() as tempdir: @@ -429,10 +571,65 @@ def test_reload_from_omx_3d_compressed(): sh.dataset.reload_from_omx_3d(blank2, [str(f)], ignore=["COUNTS"]) assert (blank2["COUNTS"].values == 0).all() np.testing.assert_array_equal(blank2["DIST"].values, expected["DIST"].values) + # An h5py handle is accepted directly and remains owned by the caller. + blank3 = xr.zeros_like(expected) + with h5py.File(f, "r") as handle: + sh.dataset.reload_from_omx_3d(blank3, handle) + assert handle.id.valid + xr.testing.assert_equal(blank3, expected) + + # The time-period dimension need not use Sharrow's default name. + custom_dims = xr.zeros_like(expected.rename(time_period="period")) + sh.dataset.reload_from_omx_3d(custom_dims, f) + xr.testing.assert_equal(custom_dims, expected.rename(time_period="period")) + + +def test_reload_from_omx_3d_shared_parallel(): + """Parallel reload fills only selected arrays and preserves source precedence.""" + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + first = Path(tempdir).joinpath("first.omx") + second = Path(tempdir).joinpath("second.omx") + _write_compressed_omx( + first, {name: data for name, data in matrices.items() if name != "TIME__PM"} + ) + _write_compressed_omx( + second, + { + "DIST": matrices["DIST"] + 1, + "TIME__PM": matrices["TIME__PM"], + }, + ) + + expected = sh.dataset.from_omx_3d( + [first, second], time_periods=["AM", "PM"], load="eager" + ).drop_vars("COUNTS") + blank = xr.zeros_like(expected) + token = "parallel-reload-" + secrets.token_hex(5) + shared = blank.shm.to_shared_memory( + token, + mode="r+", + load=False, + array_order={ + name: "last-axis-first" + for name, variable in blank.data_vars.items() + if variable.ndim == 3 + }, + ) + try: + sh.dataset.reload_from_omx_3d(shared, [first, second], workers=2) + xr.testing.assert_equal(shared, expected) + assert shared["TIME"].data[..., 0].flags.c_contiguous + finally: + shared.shm.release_shared_memory() + + with pytest.raises(ValueError, match="shared-memory-backed"): + sh.dataset.reload_from_omx_3d(blank, [first, second], workers=2) + with pytest.raises(ValueError, match="positive integer"): + sh.dataset.reload_from_omx_3d(blank, [first, second], workers=0) def test_from_omx_compressed_blosc(): - import h5py import hdf5plugin rng = np.random.default_rng(7) @@ -450,7 +647,20 @@ def test_from_omx_compressed_blosc(): out.attrs["SHAPE"] = np.asarray([25, 25], dtype=int) with h5py.File(f, mode="r") as back: ds = sh.dataset.from_omx(back, indexes="taz") + token = "blosc-skims-" + secrets.token_hex(5) + shared = sh.dataset.from_omx_3d( + f, + indexes="taz", + time_periods=["AM"], + load="shared", + workers=2, + shared_memory_key=token, + ) np.testing.assert_array_equal(ds["DIST"].values, arr) + try: + np.testing.assert_array_equal(shared["DIST"].values, arr.astype(np.float32)) + finally: + shared.shm.release_shared_memory() def test_from_omx_3d_to_zarr(): @@ -474,7 +684,7 @@ def test_from_omx_3d_writable_handle(): with tempfile.TemporaryDirectory() as tempdir: f = Path(tempdir).joinpath("skims.omx") _write_compressed_omx(f, matrices) - with openmatrix.open_file(f, mode="a") as back: + with h5py.File(f, mode="a") as back: skims = sh.dataset.from_omx_3d( back, time_periods=["AM", "PM"], max_float_precision=64 ) diff --git a/sharrow/tests/test_omx.py b/sharrow/tests/test_omx.py new file mode 100644 index 0000000..0516734 --- /dev/null +++ b/sharrow/tests/test_omx.py @@ -0,0 +1,86 @@ +from pathlib import Path + +import h5py +import numpy as np +import pytest +import xarray as xr + +from sharrow.omx import split_omx +from sharrow.translate import omx_to_zarr + + +@pytest.fixture +def omx_file(tmp_path: Path) -> tuple[Path, dict[str, np.ndarray]]: + """Create a representative OMX-format HDF5 file for utility tests.""" + matrices = { + "DIST": np.arange(9, dtype=np.float32).reshape(3, 3), + "TIME__AM": np.arange(10, 19, dtype=np.float32).reshape(3, 3), + "TIME__PM": np.arange(20, 29, dtype=np.float32).reshape(3, 3), + } + path = tmp_path / "skims.omx" + with h5py.File(path, "w") as handle: + handle.attrs["OMX_VERSION"] = np.bytes_(b"0.2") + handle.attrs["SHAPE"] = np.asarray([3, 3], dtype=np.int32) + for name, values in matrices.items(): + handle.create_dataset( + f"data/{name}", + data=values, + chunks=(2, 2), + compression="gzip", + ) + handle.create_dataset("lookup/taz", data=np.asarray([101, 102, 103])) + return path, matrices + + +def test_split_omx_with_global_lookups(omx_file, tmp_path): + """Matrix chunks retain datasets, compression, lookups, and OMX metadata.""" + source_path, matrices = omx_file + destination = tmp_path / "global-lookups" + + split_omx(source_path, destination, global_lookups=True, n_chunks=2) + + matrix_names = set() + for chunk_number in range(2): + with h5py.File(destination / f"skims-chunk{chunk_number}.omx") as handle: + assert tuple(handle.attrs["SHAPE"]) == (3, 3) + np.testing.assert_array_equal(handle["lookup/taz"], [101, 102, 103]) + for name, dataset in handle["data"].items(): + matrix_names.add(name) + assert dataset.compression == "gzip" + np.testing.assert_array_equal(dataset, matrices[name]) + assert matrix_names == set(matrices) + + +def test_split_omx_with_separate_lookups(omx_file, tmp_path): + """The default split writes each lookup to a standalone OMX file.""" + source_path, matrices = omx_file + destination = tmp_path / "separate-lookups" + + split_omx(source_path, destination) + + for name, values in matrices.items(): + with h5py.File(destination / f"{name}.omx") as handle: + assert list(handle["data"]) == [name] + assert list(handle["lookup"]) == [] + np.testing.assert_array_equal(handle[f"data/{name}"], values) + with h5py.File(destination / "_taz.omx") as handle: + assert list(handle["data"]) == [] + np.testing.assert_array_equal(handle["lookup/taz"], [101, 102, 103]) + + +def test_omx_to_zarr(omx_file, tmp_path): + """OMX conversion reads HDF5 matrices into the expected Zarr dimensions.""" + source_path, matrices = omx_file + destination = tmp_path / "skims.zarr" + + omx_to_zarr(source_path, destination, time_periods=["AM", "PM"]) + + with xr.open_zarr(destination) as dataset: + np.testing.assert_array_equal(dataset["DIST"], matrices["DIST"]) + np.testing.assert_array_equal( + dataset["TIME"].sel(time_period="AM"), matrices["TIME__AM"] + ) + np.testing.assert_array_equal( + dataset["TIME"].sel(time_period="PM"), matrices["TIME__PM"] + ) + np.testing.assert_array_equal(dataset.coords["otaz"], [101, 102, 103]) diff --git a/sharrow/translate.py b/sharrow/translate.py index 7236701..742ac78 100644 --- a/sharrow/translate.py +++ b/sharrow/translate.py @@ -1,12 +1,6 @@ import logging -import numpy as np -import pandas as pd -import xarray as xr - -from sharrow.dataset import Dataset - -from .dataset import one_based, zero_based +from .dataset import from_omx_3d logger = logging.getLogger("sharrow.translate") @@ -20,77 +14,17 @@ def omx_to_zarr( time_periods=None, time_period_sep="__", ): - try: - from larch import OMX - except ImportError: - raise ImportError("larch is required to read OMX files") from None - - bucket = {} - - r1 = r2 = None - - for omx_filename in omx_filenames: - logger.info(f"reading metadata from {omx_filename}") - - omx = OMX(omx_filename) - omx_data = omx.data - omx_shape = omx.shape - omx_lookup = omx.lookup - - data_names = list(omx_data._v_children.keys()) - n1, n2 = omx_shape - if indexes is None: - # default reads mapping if only one lookup is included, otherwise one-based - if len(omx_lookup._v_children) == 1: - ranger = None - else: - ranger = one_based - elif indexes == "one-based": - ranger = one_based - elif indexes == "zero-based": - ranger = zero_based - elif indexes in set(omx_lookup._v_children): - ranger = None - else: - raise NotImplementedError( - "only one-based, zero-based, and named indexes are implemented" - ) - if ranger is not None: - r1 = ranger(n1) - r2 = ranger(n2) - else: - r1 = r2 = pd.Index(omx_lookup[indexes]) - - if time_periods is None: - raise ValueError("must give time periods explicitly") - - bucket.update({i: omx.data[i] for i in omx.data._v_children}) - - data_names = list(bucket.keys()) + """Convert one or more OMX-format HDF5 files into a Zarr store.""" + logger.info(f"reading metadata from {omx_filenames}") + dataset = from_omx_3d( + omx_filenames, + index_names=index_names, + indexes=indexes, + time_periods=time_periods, + time_period_sep=time_period_sep, + ) logger.info(f"writing to {zarr_directory}") - for k in data_names: - logger.info(f" - {k}") - ds = Dataset() - if time_period_sep in k: - base_k, time_k = k.split(time_period_sep, 1) - ds[base_k] = xr.DataArray( - np.float32(0), - dims=index_names, - coords={ - index_names[0]: r1, - index_names[1]: r2, - index_names[2]: time_periods, - }, - ) - ds[base_k].loc[:, :, time_k] = bucket[k][:] - else: - ds[k] = xr.DataArray( - bucket[k][:], - dims=index_names[:2], - coords={ - index_names[0]: r1, - index_names[1]: r2, - }, - ) - ds.to_zarr(zarr_directory, mode="a") + for name in dataset.data_vars: + logger.info(f" - {name}") + dataset.to_zarr(zarr_directory, mode="a") diff --git a/uv.lock b/uv.lock index ba7d0af..75e692a 100644 --- a/uv.lock +++ b/uv.lock @@ -49,10 +49,10 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "msgpack", marker = "python_full_version < '3.10'" }, - { name = "ndindex", marker = "python_full_version < '3.10'" }, - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "py-cpuinfo", marker = "python_full_version < '3.10'" }, + { name = "msgpack" }, + { name = "ndindex" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, + { name = "py-cpuinfo" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f7/60/5bc8601f8ffcd5d8787b346898de8a0b454d031c3e158e3bbc312003984e/blosc2-2.5.1.tar.gz", hash = "sha256:47d5df50e7286edf81e629ece35f87f13f55c13c5e8545832188c420c75d1659", size = 4676483, upload-time = "2024-01-25T12:31:31.168Z" } wheels = [ @@ -96,14 +96,14 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "msgpack", marker = "python_full_version >= '3.10'" }, - { name = "ndindex", marker = "python_full_version >= '3.10'" }, - { name = "numexpr", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_machine != 'wasm32'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "msgpack" }, + { name = "ndindex" }, + { name = "numexpr", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'wasm32'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "py-cpuinfo", marker = "python_full_version >= '3.10' and platform_machine != 'wasm32'" }, - { name = "requests", marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "py-cpuinfo", marker = "platform_machine != 'wasm32'" }, + { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/da/6abdd7d7d9eec18a2c7f7f48a652c8679a78fc39b3b6c6723e933a65ce5f/blosc2-3.11.0.tar.gz", hash = "sha256:458a29304899000658f43411e2bfe08f6f1ea4feb4c82909002ff1b066a11886", size = 3972012, upload-time = "2025-10-28T18:03:24.305Z" } wheels = [ @@ -153,7 +153,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, + { name = "pycparser" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -355,7 +355,7 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } wheels = [ @@ -372,7 +372,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } wheels = [ @@ -499,14 +499,14 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "cloudpickle", marker = "python_full_version < '3.10'" }, - { name = "fsspec", marker = "python_full_version < '3.10'" }, - { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "partd", marker = "python_full_version < '3.10'" }, - { name = "pyyaml", marker = "python_full_version < '3.10'" }, - { name = "toolz", marker = "python_full_version < '3.10'" }, + { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" } }, + { name = "cloudpickle" }, + { name = "fsspec" }, + { name = "importlib-metadata" }, + { name = "packaging" }, + { name = "partd" }, + { name = "pyyaml" }, + { name = "toolz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/2e/568a422d907745a3a897a732c83d05a3923d8cfa2511a6abea2a0e19994e/dask-2024.8.0.tar.gz", hash = "sha256:f1fec39373d2f101bc045529ad4e9b30e34e6eb33b7aa0fa7073aec7b1bf9eee", size = 9895684, upload-time = "2024-08-06T20:23:54.464Z" } wheels = [ @@ -523,14 +523,14 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "click", version = "8.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "cloudpickle", marker = "python_full_version >= '3.10'" }, - { name = "fsspec", marker = "python_full_version >= '3.10'" }, - { name = "importlib-metadata", marker = "python_full_version >= '3.10' and python_full_version < '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "partd", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, - { name = "toolz", marker = "python_full_version >= '3.10'" }, + { name = "click", version = "8.3.0", source = { registry = "https://pypi.org/simple" } }, + { name = "cloudpickle" }, + { name = "fsspec" }, + { name = "importlib-metadata", marker = "python_full_version < '3.12'" }, + { name = "packaging" }, + { name = "partd" }, + { name = "pyyaml" }, + { name = "toolz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/f0/d747f9517f2a50b835513da36a6da0cffa7d1f0a8b33f60e642ff78879a8/dask-2025.10.0.tar.gz", hash = "sha256:fd3159c319c27cea39b891c0f22d60056a33575fb4906618eab0aeeb5dcd0cbc", size = 10974677, upload-time = "2025-10-14T19:50:36.556Z" } wheels = [ @@ -584,7 +584,7 @@ name = "donfig" version = "0.8.1.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyyaml", marker = "python_full_version >= '3.11'" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } wheels = [ @@ -596,7 +596,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -673,7 +673,7 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" } wheels = [ @@ -714,7 +714,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" } @@ -800,7 +800,7 @@ name = "importlib-metadata" version = "8.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "python_full_version < '3.12'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } wheels = [ @@ -841,19 +841,19 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "appnope", marker = "python_full_version < '3.10' and sys_platform == 'darwin'" }, - { name = "comm", marker = "python_full_version < '3.10'" }, - { name = "debugpy", marker = "python_full_version < '3.10'" }, - { name = "ipython", version = "8.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "jupyter-client", marker = "python_full_version < '3.10'" }, - { name = "jupyter-core", version = "5.8.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.10'" }, - { name = "nest-asyncio", marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "psutil", marker = "python_full_version < '3.10'" }, - { name = "pyzmq", marker = "python_full_version < '3.10'" }, - { name = "tornado", marker = "python_full_version < '3.10'" }, - { name = "traitlets", marker = "python_full_version < '3.10'" }, + { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython", version = "8.18.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jupyter-client" }, + { name = "jupyter-core", version = "5.8.1", source = { registry = "https://pypi.org/simple" } }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/1d/d5ba6edbfe6fae4c3105bca3a9c889563cc752c7f2de45e333164c7f4846/ipykernel-6.31.0.tar.gz", hash = "sha256:2372ce8bc1ff4f34e58cafed3a0feb2194b91fc7cad0fc72e79e47b45ee9e8f6", size = 167493, upload-time = "2025-10-20T11:42:39.948Z" } wheels = [ @@ -870,20 +870,20 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "appnope", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, - { name = "comm", marker = "python_full_version >= '3.10'" }, - { name = "debugpy", marker = "python_full_version >= '3.10'" }, - { name = "ipython", version = "8.37.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython", version = "8.37.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "ipython", version = "9.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "jupyter-client", marker = "python_full_version >= '3.10'" }, - { name = "jupyter-core", version = "5.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.10'" }, - { name = "nest-asyncio", marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "psutil", marker = "python_full_version >= '3.10'" }, - { name = "pyzmq", marker = "python_full_version >= '3.10'" }, - { name = "tornado", marker = "python_full_version >= '3.10'" }, - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "jupyter-client" }, + { name = "jupyter-core", version = "5.9.1", source = { registry = "https://pypi.org/simple" } }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/a4/4948be6eb88628505b83a1f2f40d90254cab66abf2043b3c40fa07dfce0f/ipykernel-7.1.0.tar.gz", hash = "sha256:58a3fc88533d5930c3546dc7eac66c6d288acde4f801e2001e65edc5dc9cf0db", size = 174579, upload-time = "2025-10-27T09:46:39.471Z" } wheels = [ @@ -898,17 +898,17 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.10'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "jedi", marker = "python_full_version < '3.10'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.10'" }, - { name = "pexpect", marker = "python_full_version < '3.10' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "stack-data", marker = "python_full_version < '3.10'" }, - { name = "traitlets", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/b9/3ba6c45a6df813c09a48bac313c22ff83efa26cbb55011218d925a46e2ad/ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27", size = 5486330, upload-time = "2023-11-27T09:58:34.596Z" } wheels = [ @@ -923,17 +923,17 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version == '3.10.*' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version == '3.10.*'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "jedi", marker = "python_full_version == '3.10.*'" }, - { name = "matplotlib-inline", marker = "python_full_version == '3.10.*'" }, - { name = "pexpect", marker = "python_full_version == '3.10.*' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version == '3.10.*'" }, - { name = "pygments", marker = "python_full_version == '3.10.*'" }, - { name = "stack-data", marker = "python_full_version == '3.10.*'" }, - { name = "traitlets", marker = "python_full_version == '3.10.*'" }, - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/85/31/10ac88f3357fc276dc8a64e8880c82e80e7459326ae1d0a211b40abf6665/ipython-8.37.0.tar.gz", hash = "sha256:ca815841e1a41a1e6b73a0b08f3038af9b2252564d01fc405356d34033012216", size = 5606088, upload-time = "2025-05-31T16:39:09.613Z" } wheels = [ @@ -949,17 +949,17 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2a/34/29b18c62e39ee2f7a6a3bba7efd952729d8aadd45ca17efc34453b717665/ipython-9.6.0.tar.gz", hash = "sha256:5603d6d5d356378be5043e69441a072b50a5b33b4503428c77b04cb8ce7bc731", size = 4396932, upload-time = "2025-09-29T10:55:53.948Z" } wheels = [ @@ -971,7 +971,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -1046,9 +1046,9 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pywin32", marker = "python_full_version < '3.10' and platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, - { name = "traitlets", marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" } }, + { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/99/1b/72906d554acfeb588332eaaa6f61577705e9ec752ddb486f302dafa292d9/jupyter_core-5.8.1.tar.gz", hash = "sha256:0a5f9706f70e64786b75acba995988915ebd4601c8a52e534a40b51c95f59941", size = 88923, upload-time = "2025-05-27T07:38:16.655Z" } wheels = [ @@ -1065,8 +1065,8 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } wheels = [ @@ -1399,8 +1399,8 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "llvmlite", version = "0.43.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "llvmlite", version = "0.43.0", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/3c/93/2849300a9184775ba274aba6f82f303343669b0592b7bb0849ea713dabb0/numba-0.60.0.tar.gz", hash = "sha256:5df6158e5584eece5fc83294b949fd30b9f1125df7708862205217e068aabf16", size = 2702171, upload-time = "2024-06-13T18:11:19.869Z" } wheels = [ @@ -1436,8 +1436,8 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "llvmlite", version = "0.45.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "llvmlite", version = "0.45.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/20/33dbdbfe60e5fd8e3dbfde299d106279a33d9f8308346022316781368591/numba-0.62.1.tar.gz", hash = "sha256:7b774242aa890e34c21200a1fc62e5b5757d5286267e71103257f4e2af0d5161", size = 2749817, upload-time = "2025-09-29T10:46:31.551Z" } @@ -1472,7 +1472,7 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/1b/1f1d880e29e719c7c6205065d1afbc91114c0d91935ac419faa43e5e08b0/numcodecs-0.12.1.tar.gz", hash = "sha256:05d91a433733e7eef268d7e80ec226a0232da244289614a8f3826901aec1098e", size = 4091415, upload-time = "2023-10-18T00:47:07.456Z" } wheels = [ @@ -1502,7 +1502,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/85/56/8895a76abe4ec94ebd01eeb6d74f587bc4cddd46569670e1402852a5da13/numcodecs-0.13.1.tar.gz", hash = "sha256:a3cf37881df0898f3a9c0d4477df88133fe85185bffe57ba31bcc2fa207709bc", size = 5955215, upload-time = "2024-10-09T16:28:00.188Z" } wheels = [ @@ -1533,8 +1533,8 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f6/48/6188e359b90a9d8a1850f2bc888c023e66f4a8b2b496820babbea414f008/numcodecs-0.16.3.tar.gz", hash = "sha256:53d705865faaf0a7927c973af3777532001c8fbb653de119c1e844608614d799", size = 6275704, upload-time = "2025-09-18T18:54:57.221Z" } wheels = [ @@ -1557,7 +1557,7 @@ wheels = [ [package.optional-dependencies] crc32c = [ - { name = "crc32c", marker = "python_full_version >= '3.11'" }, + { name = "crc32c" }, ] [[package]] @@ -1568,7 +1568,7 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/21/67/c7415cf04ebe418193cfd6595ae03e3a64d76dac7b9c010098b39cc7992e/numexpr-2.10.2.tar.gz", hash = "sha256:b0aff6b48ebc99d2f54f27b5f73a58cb92fde650aeff1b397c71c8788b4fff1a", size = 106787, upload-time = "2024-11-23T13:34:23.127Z" } wheels = [ @@ -1619,7 +1619,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/2f/fdba158c9dbe5caca9c3eca3eaffffb251f2fb8674bf8e2d0aed5f38d319/numexpr-2.14.1.tar.gz", hash = "sha256:4be00b1086c7b7a5c32e31558122b7b80243fe098579b170967da83f3152b48b", size = 119400, upload-time = "2025-10-13T16:17:27.351Z" } @@ -1887,23 +1887,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/8e/2844c3959ce9a63acc7c8e50881133d86666f0420bcde695e115ced0920f/numpy-2.3.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:81b3a59793523e552c4a96109dde028aa4448ae06ccac5a76ff6532a85558a7f", size = 12973130, upload-time = "2025-10-15T16:18:09.397Z" }, ] -[[package]] -name = "openmatrix" -version = "0.3.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "tables", version = "3.9.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "tables", version = "3.10.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "tables", version = "3.10.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/a4/c890e3af726ab91e496017fb108f6335663d6936cf6a26b17e66776e1b4c/OpenMatrix-0.3.5.0.tar.gz", hash = "sha256:1c10155bbe65eb4f542c388813d8dad8f317dfe642fb4f616276c95e6654bdee", size = 11626, upload-time = "2019-12-21T17:52:58.945Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/e8/6a8227e97127a2a1db98f5a22ca565bfe942e89de0644837627920101213/OpenMatrix-0.3.5.0-py3-none-any.whl", hash = "sha256:eeeb1a628b5eeefb572361537df5ad6e4d12c3cda1233023c4c367815cab6b5f", size = 13003, upload-time = "2019-12-21T17:52:57.777Z" }, -] - [[package]] name = "packaging" version = "25.0" @@ -2510,9 +2493,9 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "attrs", marker = "python_full_version < '3.10'" }, - { name = "rpds-py", version = "0.27.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "attrs" }, + { name = "rpds-py", version = "0.27.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } wheels = [ @@ -2529,9 +2512,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "attrs", marker = "python_full_version >= '3.10'" }, - { name = "rpds-py", version = "0.28.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "attrs" }, + { name = "rpds-py", version = "0.28.0", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -2543,10 +2526,10 @@ name = "requests" version = "2.32.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi", marker = "python_full_version >= '3.10'" }, - { name = "charset-normalizer", marker = "python_full_version >= '3.10'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "urllib3", marker = "python_full_version >= '3.10'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ @@ -2854,7 +2837,7 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ae/00/48c2f661e2816ccf2ecd77982f6605b2950afe60f60a52b4cbbc2504aa8f/scipy-1.13.1.tar.gz", hash = "sha256:095a87a0312b08dfd6a6155cbbd310a8c51800fc931b8c0b84003014b874ed3c", size = 57210720, upload-time = "2024-05-23T03:29:26.079Z" } @@ -2894,7 +2877,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/0a/ca/d8ace4f98322d01abcd52d381134344bf7b431eba7ed8b42bdea5a3c2ac9/scipy-1.16.3.tar.gz", hash = "sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb", size = 30597883, upload-time = "2025-10-28T17:38:54.068Z" } wheels = [ @@ -2996,7 +2979,6 @@ dev = [ { name = "ipykernel", version = "6.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "ipykernel", version = "7.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "nbmake" }, - { name = "openmatrix" }, { name = "pytest" }, { name = "pytest-regressions" }, { name = "scipy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -3028,7 +3010,6 @@ requires-dist = [ dev = [ { name = "ipykernel", specifier = ">=6.31.0" }, { name = "nbmake", specifier = ">=1.5.5" }, - { name = "openmatrix", specifier = ">=0.3.5.0" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-regressions", specifier = ">=2.8.3" }, { name = "scipy", specifier = ">=1.13.1" }, @@ -3053,9 +3034,9 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "numba", version = "0.60.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "scipy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "numba", version = "0.60.0", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.13.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/8c/97/ef70784ece46b7071f357f89f69ae8b120f7a2dff1a70f9747563b709683/sparse-0.15.5.tar.gz", hash = "sha256:4c76ce0c96f5cd5c31b7e79e650f0022424c2b16f05f10049e9c6381ee4be266", size = 359719, upload-time = "2025-01-15T14:01:19.568Z" } wheels = [ @@ -3072,8 +3053,8 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "numba", version = "0.62.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "numba", version = "0.62.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/63/74/5c674277fc3d61bd1863d233a8e1f7ddf35cb1adeeaf9973888629e7a9b1/sparse-0.17.0.tar.gz", hash = "sha256:6b1ad51a810c5be40b6f95e28513ec810fe1c785923bd83b2e4839a751df4bf7", size = 642387, upload-time = "2025-05-20T08:17:03.256Z" } @@ -3095,113 +3076,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] -[[package]] -name = "tables" -version = "3.9.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "blosc2", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "numexpr", version = "2.10.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "py-cpuinfo", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/31/83/8a13be8338219c3fe0aa7357d1ec4edb27bc346e0f224df7212892b243b5/tables-3.9.2.tar.gz", hash = "sha256:d470263c2e50c4b7c8635a0d99ac1ff2f9e704c24d71e5fa33c4529e7d0ad9c3", size = 4683437, upload-time = "2023-11-27T11:53:17.229Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/95/dea498a083e1fbe60bf0ea999da38cb8e33b5506ba99237b5416267a9bc0/tables-3.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8a4e71fc9d2a3a0cacce4994afd47cd5f4797093ff9cee2cc7dc87e51f308107", size = 7392824, upload-time = "2023-11-27T11:52:19.054Z" }, - { url = "https://files.pythonhosted.org/packages/49/b0/e424d4d409925cdf38689bb85f33e2a7c7bca310fb188de65af031c933b7/tables-3.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbea426ce9bdd60cda435a265823b31d18f2b36e9045fb2d565679825a7aa46", size = 7113863, upload-time = "2023-11-27T11:52:24.217Z" }, - { url = "https://files.pythonhosted.org/packages/22/86/5a474f002126e37d9ee1062e71dc99223d1618ffcfbd5683a7662766c4af/tables-3.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e756c272bb111d016fec1d03a60095403a8fb42a5fbaf5f317dcf6e3b9d8e92e", size = 7481194, upload-time = "2023-11-27T11:52:29.369Z" }, - { url = "https://files.pythonhosted.org/packages/90/eb/8b998ae92009d6e9a696d8ec43a5b5ab9200a715f8da86d9f952a1a1bde5/tables-3.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:eea41cb32dd22b30d6f3dd4e113f6d693384d301c89f3c4b4712f90c9c955875", size = 4391242, upload-time = "2023-11-27T11:52:33.314Z" }, - { url = "https://files.pythonhosted.org/packages/14/41/ea4928ece6c79614d1b61215a52760d88a134e55eca42a3a09815468afa9/tables-3.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d71913fb8147dc6132595b94fc82f88f6c2436a3b5c57aadfe26c680f96aa387", size = 7323512, upload-time = "2023-11-27T11:52:37.305Z" }, - { url = "https://files.pythonhosted.org/packages/d2/de/a3f2a4c6ac32b81e4e2789d7364db0f8fff0657f1b767004c3e63b353142/tables-3.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d6bbc477d038a17c5062ab6ccd94c8b1fa365cf017b9a2ad6c2dff1a07abb2b", size = 7112660, upload-time = "2023-11-27T11:52:40.427Z" }, - { url = "https://files.pythonhosted.org/packages/03/76/53d432936c6955de1346b629b9f92213b11ee9e8aeab8831027fac9e053d/tables-3.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e67c71070b871fade3694a4c764504e03836bb1843321766cf2e40b7d280e84", size = 7478756, upload-time = "2023-11-27T11:52:44.182Z" }, - { url = "https://files.pythonhosted.org/packages/64/09/1071a5cef60108dc3fd8d44596371d223834171800f43a652a9bd523588f/tables-3.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:ab9291ff4d243e7966b6706a2675b83138bd9bbe82721d695b78971660d59632", size = 4391764, upload-time = "2023-11-27T11:52:47.4Z" }, - { url = "https://files.pythonhosted.org/packages/01/4f/cb9e44384982889ff720b8fabddb24aa4b0deb69afb1a5f09dd17a93b4ca/tables-3.9.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c14dc879b041cf53be1afe9e5ed581e1aeacdcee9e2e1ee79110dc96a4c8d97c", size = 7351431, upload-time = "2023-11-27T11:52:51.371Z" }, - { url = "https://files.pythonhosted.org/packages/9a/e0/070c3ebc11614f1c3fc413f2327ea9d1818444eab7ae4975a2fea7de03cb/tables-3.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2848fb3dce30a7b83fa099d026a91d7b10ad48afae04fa10f974f1da3f1e2bbf", size = 7077135, upload-time = "2023-11-27T11:52:54.361Z" }, - { url = "https://files.pythonhosted.org/packages/dd/50/8efbd7339a1fd69658f623bcee5e4c0318d823cc39bb23dbc6b0ff1a7491/tables-3.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b131c9b4e003816a45e2efe5c5c797d01d8308cac4aee72597a15837cedb605c", size = 7442059, upload-time = "2023-11-27T11:52:57.999Z" }, - { url = "https://files.pythonhosted.org/packages/8b/84/93dd082a991c36f3e55e170cb71c4011256e45549c0a3b0d3693cf9053e3/tables-3.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:c6304d321452fd56865e5c309e38373011b0f0f6c714786c5660613ceb623acb", size = 4374051, upload-time = "2023-11-27T11:53:00.801Z" }, - { url = "https://files.pythonhosted.org/packages/64/5d/eb0c1b4401cbd9006589cb1f2a4288e70faa246538c7115ee7e54805c2d7/tables-3.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c52087ed8b90a5f6ba87f0adcd1c433e5f5db7c7ca5984b08ff45f2247635f7d", size = 7396210, upload-time = "2023-11-27T11:53:03.627Z" }, - { url = "https://files.pythonhosted.org/packages/a9/9c/fb98d7a5b9e1c7f588bf71ea75c87711212c964f0c1c8e0cb1246bda0a35/tables-3.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:164b945d0cb731c7232775fd3657f150bcf05413928b86033b023a1dc8dbeb05", size = 7116751, upload-time = "2023-11-27T11:53:07.406Z" }, - { url = "https://files.pythonhosted.org/packages/62/cf/897ad10477897bd55bcb3431833b274adda1c01bc8f856b8de57f6de0b83/tables-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a068d4ad08d5a6b2ad457f60ac6676efdab9e29459e776e433d5537a46e62e41", size = 7483600, upload-time = "2023-11-27T11:53:10.155Z" }, - { url = "https://files.pythonhosted.org/packages/28/67/f710befc6a57a339ac0aa7272c7df58014fff4f8863c16f683894b8c7893/tables-3.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:bca5a6bf162a84a6ef74ca4017b28c59c1526cffdbd93ce94c98ff8f9593f1d5", size = 4394103, upload-time = "2023-11-27T11:53:13.632Z" }, -] - -[[package]] -name = "tables" -version = "3.10.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.10.*'", -] -dependencies = [ - { name = "blosc2", version = "3.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "numexpr", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "packaging", marker = "python_full_version == '3.10.*'" }, - { name = "py-cpuinfo", marker = "python_full_version == '3.10.*'" }, - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/5d/96708a84e9fcd29d1f684d56d4c38a23d29b1c934599a072a49f27ccfa71/tables-3.10.1.tar.gz", hash = "sha256:4aa07ac734b9c037baeaf44aec64ec902ad247f57811b59f30c4e31d31f126cf", size = 4762413, upload-time = "2024-08-17T09:57:47.127Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/69/a768ec8104ada032c9be09f521f548766ddd0351bc941c9d42fa5db001de/tables-3.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bca9d11a570ca1bc57f0845e54e55c3093d5a1ace376faee639e09503a73745b", size = 6823691, upload-time = "2024-08-17T09:56:50.229Z" }, - { url = "https://files.pythonhosted.org/packages/e4/2d/074bc14b39de9b552eec02ee583eff2997d903da1355f4450506335a6055/tables-3.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b62881cb682438d1e92b9178db42b160638aef3ca23341f7d98e9b27821b1eb4", size = 5471221, upload-time = "2024-08-17T09:56:54.84Z" }, - { url = "https://files.pythonhosted.org/packages/4a/30/29411ab804b5ac4bee25c82ba38f4e7a8c0b52c6a1cdbeea7d1db33a53fe/tables-3.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9cf1bfd8b0e0195196205fc8a134628219cff85d20da537facd67a291e6b347", size = 7170201, upload-time = "2024-08-17T09:56:59.011Z" }, - { url = "https://files.pythonhosted.org/packages/0a/7d/3165c7538b8e89b22fa17ad68e04106cca7023cf68e94011ae7b3b6d2a78/tables-3.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77f0e6dd45b91d99bf3976c8655c48fe3816baf390b9098e4fb2f0fdf9da7078", size = 7571035, upload-time = "2024-08-17T09:57:03.115Z" }, - { url = "https://files.pythonhosted.org/packages/46/b3/985a23d2cf27aad383301a5e99e1851228a1941b868515612b5357bded5f/tables-3.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:d90542ec172d1d60df0b796c48ad446f2b69a5d5cd3077bd6450891b854d1ffb", size = 6311650, upload-time = "2024-08-17T09:57:06.593Z" }, - { url = "https://files.pythonhosted.org/packages/dc/04/957264eb35e60251830a965e2d02332eb36ed14fbd8345df06981bbf3ece/tables-3.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8917262a2bb3cd79d37e108557e34ec4b365fdcc806e01dd10765a84c65dab6", size = 6790492, upload-time = "2024-08-17T09:57:10.247Z" }, - { url = "https://files.pythonhosted.org/packages/b2/19/eb7af9d92aaf6766f5fedfce11a97ab03cf39856561c5f562dc0c769a682/tables-3.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f93f6db623b484bb6606537c2a71e95ee34fae19b0d891867642dd8c7be05af6", size = 5506835, upload-time = "2024-08-17T09:57:13.883Z" }, - { url = "https://files.pythonhosted.org/packages/b0/8f/897324e1ad543ca439b2c91f04c406f3eeda6e7ff2f43b4cd939f05043e4/tables-3.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01ca51624bca1a87e703d6d6b796368bc3460ff007ea8b1341be03bedd863833", size = 7166960, upload-time = "2024-08-17T09:57:17.463Z" }, - { url = "https://files.pythonhosted.org/packages/4e/5c/3f21d1135bf60af99ac79a17bbffd333d69763df2197ba04f47dd30bbd4e/tables-3.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9372516c76be3a05a573df63a69ce38315d03b5816d2a1e89c48129ec8b161b0", size = 7568724, upload-time = "2024-08-17T09:57:23.02Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e3/3ee6b66263902eccadc4e0e23bca7fb480fd190904b7ce0bea4777b5b799/tables-3.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:09190fb504888aeacafb7739c13d5c5a3e87af3d261f4d2f832b1f8407be133a", size = 6312200, upload-time = "2024-08-17T09:57:26.322Z" }, - { url = "https://files.pythonhosted.org/packages/95/ec/ea6c476e33602c172c797fe8f8ab96d007d964137068276d142b142a28e5/tables-3.10.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7090af37909e3bf229d5599fa442633e5a93b6082960b01038dc0106e07a8da", size = 6791597, upload-time = "2024-08-17T09:57:29.598Z" }, - { url = "https://files.pythonhosted.org/packages/74/02/a967a506e9204e3328a8c03f67e6f3c919defc8df11aba83ae5b2abf7b0f/tables-3.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:203ed50c0c5f30f007df7633089b2a567b99856cd25d68f19d91624a8db2e7ad", size = 5474779, upload-time = "2024-08-17T09:57:32.43Z" }, - { url = "https://files.pythonhosted.org/packages/c3/26/925793f753664ec698b2c6315c818269313db143da38150897cf260405c2/tables-3.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e36ce9f10471c69c1f0b06c6966de762558a35d62592c55df7994a8019adaf0c", size = 7130683, upload-time = "2024-08-17T09:57:36.181Z" }, - { url = "https://files.pythonhosted.org/packages/d8/79/2b34f22284459e940a84e71dba19b2a34c7cc0ce3cdf685923c50d5b9611/tables-3.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f233e78cc9fa4157ec4c3ef2abf01a731fe7969bc6ed73539e5f4cd3b94c98b2", size = 7531367, upload-time = "2024-08-17T09:57:39.864Z" }, - { url = "https://files.pythonhosted.org/packages/3d/27/5a23830f611e26dd7ee104096c6bb82e481b16f3f17ccaed3075f8d48312/tables-3.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:34357d2f2f75843a44e6fe54d1f11fc2e35a8fd3cb134df3d3362cff78010adb", size = 6295046, upload-time = "2024-08-17T09:57:43.561Z" }, - { url = "https://files.pythonhosted.org/packages/d3/d4/e7c25df877e054b05f146d6ccb920bcdbe8d39b35a0962868b80547532c7/tables-3.10.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6fc5b46a4f359249c3ab9a0a0a2448d7e680e68cffd63fdf3fb7171781edd46e", size = 6824253, upload-time = "2024-11-09T19:26:06.428Z" }, - { url = "https://files.pythonhosted.org/packages/c6/49/091865d75090a24493bd1b66e52d72f4d9627ff42983a13d4dcd89455d02/tables-3.10.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2ecabd7f459d40b7f9f5256850dd5f43773fda7b789f827de92c3d26df1e320f", size = 5499587, upload-time = "2024-11-09T19:26:12.402Z" }, - { url = "https://files.pythonhosted.org/packages/23/83/9dac8af333149fa01add439f710d4a312b70faf81c2f59a16b8bfaebb75e/tables-3.10.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40a4ee18f3c9339d9dd8fd3777c75cda5768f2ff347064a2796f59161a190af8", size = 7128236, upload-time = "2024-11-09T19:26:15.716Z" }, - { url = "https://files.pythonhosted.org/packages/89/fd/62f31643596f6ab71fc6d2a87acdee0bc01a03fbe1a7f3f6dc0c91e2546d/tables-3.10.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757c6ea257c174af8036cf8f273ede756bbcd6db5ac7e2a4d64e788b0f371152", size = 7527953, upload-time = "2024-11-09T19:26:20.229Z" }, -] - -[[package]] -name = "tables" -version = "3.10.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", -] -dependencies = [ - { name = "blosc2", version = "3.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "numexpr", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "py-cpuinfo", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/15/50/23ead25f60bb1babe7f2f061d8a2f8c2f6804c1a20b3058677beb9085b56/tables-3.10.2.tar.gz", hash = "sha256:2544812a7186fadba831d6dd34eb49ccd788d6a83f4e4c2b431b835b6796c910", size = 4779722, upload-time = "2025-01-04T20:44:13.034Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/f6/ef0c376c1fa01b916d5db0c2681be063f6289ee99faf7bb6610e0b55b773/tables-3.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:63f8adec3c4421a011c5c6a245c0c1fccf16dba7aaa67d9915d2821cf365ed4a", size = 6767194, upload-time = "2025-01-04T20:42:53.5Z" }, - { url = "https://files.pythonhosted.org/packages/d9/d0/accd41382fa9da45bf816c56f85bda64223a3b8d0006d3496b67e0781a6e/tables-3.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34c120bff666d33d3bdfb9e33173a4869d5f34e6c87824f2c7ec6a72c8dfab82", size = 5482665, upload-time = "2025-01-04T20:42:58.589Z" }, - { url = "https://files.pythonhosted.org/packages/59/2f/c95e94423c463177b8a7d55a1dbbd524840fe6a684844ff728f238e71f68/tables-3.10.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e71f63ac67c583ac42943c99c2d33bcc9e361e94d1ab1a763dc0698bdd9ff815", size = 7117696, upload-time = "2025-01-04T20:43:04.014Z" }, - { url = "https://files.pythonhosted.org/packages/88/d5/71665919aa2a5a3d2a20eeef3c71dc7c2ebbd9f26d114a7808514aba24d6/tables-3.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:154773f97763ccc91a29bcead6ab7b5ef164c2ed8c409cd79a2115aa9b4184c9", size = 7520921, upload-time = "2025-01-04T20:43:10.002Z" }, - { url = "https://files.pythonhosted.org/packages/46/96/b5023c1f7b9d560cac3e2c0daceebaeb88dd24c70c75db2d291abfa563e5/tables-3.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:96b5e945d275415e79ddb0578657ecc6ac77030dcc0632ab2c39f89390bb239d", size = 6407137, upload-time = "2025-01-04T20:43:15.838Z" }, - { url = "https://files.pythonhosted.org/packages/ab/c4/1efbcc699db863d88874f3d111e5bb6dd2e0fbaca38f91c992e696324730/tables-3.10.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c6ba58205d1f6a4e0e2212bc221e76cf104f22190f90c3f1683f3c1ab138f28f", size = 6734990, upload-time = "2025-01-04T20:43:20.794Z" }, - { url = "https://files.pythonhosted.org/packages/4a/db/4c7facfc805ab764f2ee256011d20f96791d2426afa3389ca7ff2a8a4ea8/tables-3.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdb5c040aa43e5e96259d6f6bb9df5b66fef2b071a6eb035c21bf6508e865d40", size = 5483377, upload-time = "2025-01-04T20:43:25.923Z" }, - { url = "https://files.pythonhosted.org/packages/93/0a/53815b516a2465b329e5dc2079c99a8b6b1a23f6b9ce5da8a7ebc7892bf4/tables-3.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e694123fa886d9be57f55fc7e1dcacac49f0b4ed4a931c795bd8f82f7111b5a8", size = 7081356, upload-time = "2025-01-04T20:43:31.066Z" }, - { url = "https://files.pythonhosted.org/packages/d3/e1/3f4adfc83eb7390abb964682a7d1df0dbe451dd2cee99750b1c7ca8e2c9d/tables-3.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6c12d0d04de89297763923ebeaddfd7e0b51f29041895db284fd4913e7448b7", size = 7483570, upload-time = "2025-01-04T20:43:36.694Z" }, - { url = "https://files.pythonhosted.org/packages/9a/d4/0b9ba57a5a8d2d05d1108055a8d70a4b066db4ebed61921de34043a31bdb/tables-3.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:a406d5dbbcb6604bd1ca129af337e0790d4e02d29d06159ddb9f74e38d756d32", size = 6388443, upload-time = "2025-01-04T20:43:42.503Z" }, - { url = "https://files.pythonhosted.org/packages/ab/02/8c7aeaa6c8aac8e0298d40dc5fc55477fddc30cb31e4dc7e5e473be4b464/tables-3.10.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7b8bc07c715bad3d447ed8f834388ef2e10265e2c4af6b1297fc61adb645948f", size = 6725764, upload-time = "2025-01-04T20:43:48.171Z" }, - { url = "https://files.pythonhosted.org/packages/91/f4/8683395d294b9e4576fd7d888aa6cf5583c013c2c0a2e47f862c2842407f/tables-3.10.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:28677ed8e1a371471495599078f48da0850f82457d6c852ca77959c974371140", size = 5442663, upload-time = "2025-01-04T20:43:53.722Z" }, - { url = "https://files.pythonhosted.org/packages/72/9b/ea43159eed8f81bfa1ead8fa8201a3c352e84c7220e046bb548736833951/tables-3.10.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaaea478dcf27dd54679ef2643c26d3b8b15676ad81e4d80a88fd1682d23deb1", size = 7078747, upload-time = "2025-01-04T20:43:59.596Z" }, - { url = "https://files.pythonhosted.org/packages/04/95/b3e88edc674e35d9011b168df0d7a9b1c3ab98733fa26e740ac7964edc2f/tables-3.10.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5e67a9f901842f9a4b1f3d2307f4bdd94047514fe0d0c558ed19c11f53c402a", size = 7479985, upload-time = "2025-01-04T20:44:04.13Z" }, - { url = "https://files.pythonhosted.org/packages/63/ca/eaa029a43d269bdda6985931d6cfd479e876cd8cf7c887d818bef05ef03b/tables-3.10.2-cp313-cp313-win_amd64.whl", hash = "sha256:5637fdcded5ba5426aa24e0e42d6f990926a4da7f193830df131dfcb7e842900", size = 6385562, upload-time = "2025-01-04T20:44:08.196Z" }, -] - [[package]] name = "tomli" version = "2.3.0" @@ -3332,9 +3206,9 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pandas", marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pandas" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/e8/8ee12706df0d34ad04b3737621a73432458d47bc8abfbd6f049e51ca89c3/xarray-2024.7.0.tar.gz", hash = "sha256:4cae512d121a8522d41e66d942fb06c526bc1fd32c2c181d5fe62fe65b671638", size = 3728663, upload-time = "2024-07-30T08:31:45.48Z" } wheels = [ @@ -3349,9 +3223,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "packaging", marker = "python_full_version == '3.10.*'" }, - { name = "pandas", marker = "python_full_version == '3.10.*'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pandas" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/ec/e50d833518f10b0c24feb184b209bb6856f25b919ba8c1f89678b930b1cd/xarray-2025.6.1.tar.gz", hash = "sha256:a84f3f07544634a130d7dc615ae44175419f4c77957a7255161ed99c69c7c8b0", size = 3003185, upload-time = "2025-06-12T03:04:09.099Z" } wheels = [ @@ -3367,9 +3241,9 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "pandas", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pandas" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7b/ce/f5dd613ddd0b3f839c59e6c2fa20c62469bf671bf4c92a12b09dc0972326/xarray-2025.10.1.tar.gz", hash = "sha256:3c2b5ad7389825bd624ada5ff26b01ac54b1aae72e2fe0d724d81d40a2bf5785", size = 3058736, upload-time = "2025-10-07T20:25:56.708Z" } wheels = [ @@ -3384,10 +3258,10 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "asciitree", marker = "python_full_version < '3.10'" }, - { name = "fasteners", marker = "python_full_version < '3.10' and sys_platform != 'emscripten'" }, - { name = "numcodecs", version = "0.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "asciitree" }, + { name = "fasteners", marker = "sys_platform != 'emscripten'" }, + { name = "numcodecs", version = "0.12.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/00/ac5c518ff1c1b1cc87a62f86ad9d19c647c19d969a91faa40d3b6342ccaa/zarr-2.18.2.tar.gz", hash = "sha256:9bb393b8a0a38fb121dbb913b047d75db28de9890f6d644a217a73cf4ae74f47", size = 3603055, upload-time = "2024-05-26T19:23:24.131Z" } wheels = [ @@ -3402,10 +3276,10 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "asciitree", marker = "python_full_version == '3.10.*'" }, - { name = "fasteners", marker = "python_full_version == '3.10.*' and sys_platform != 'emscripten'" }, - { name = "numcodecs", version = "0.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "asciitree" }, + { name = "fasteners", marker = "sys_platform != 'emscripten'" }, + { name = "numcodecs", version = "0.13.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/c4/187a21ce7cf7c8f00c060dd0e04c2a81139bb7b1ab178bba83f2e1134ce2/zarr-2.18.3.tar.gz", hash = "sha256:2580d8cb6dd84621771a10d31c4d777dca8a27706a1a89b29f42d2d37e2df5ce", size = 3603224, upload-time = "2024-09-04T23:20:16.595Z" } wheels = [ @@ -3421,11 +3295,11 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "donfig", marker = "python_full_version >= '3.11'" }, - { name = "numcodecs", version = "0.16.3", source = { registry = "https://pypi.org/simple" }, extra = ["crc32c"], marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, + { name = "donfig" }, + { name = "numcodecs", version = "0.16.3", source = { registry = "https://pypi.org/simple" }, extra = ["crc32c"] }, + { name = "numpy", version = "2.3.4", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d6/67/14be68a7bad15eecda09b1e81fca2420f7533645fe187bf4d6104c1aad52/zarr-3.1.3.tar.gz", hash = "sha256:01342f3e26a02ed5670db608a5576fbdb8d76acb5c280bd2d0082454b1ba6f79", size = 349125, upload-time = "2025-09-18T19:32:41.688Z" } wheels = [