diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index ccf17239b..96d6d67f3 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -15,7 +15,7 @@ from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Iterator, Sequence from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime from enum import Enum from fnmatch import fnmatchcase @@ -52,7 +52,7 @@ ) from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError from .keyring_utils import keyring_lookup, keyring_save -from .misctypes import Digest, RemoteReadableAsset +from .misctypes import BasePath, Digest, RemoteReadableAsset from .utils import ( USER_AGENT, check_dandi_version, @@ -1426,6 +1426,20 @@ def get_assets(self, order: str | None = None) -> Iterator[RemoteAsset]: f"No such version: {self.version_id!r} of Dandiset {self.identifier}" ) + def get_path(self, path: str = "") -> RemoteDandisetPath: + """Return a lazy path for browsing this version's asset directories. + + Listing a directory uses the paginated `/assets/paths/` endpoint. + Listed children retain their type, recursive count and size, so reading + those properties makes no further requests. An arbitrary unlisted path + requires a listing of its parent to determine whether it exists. + + Path objects cache listings; call this method again to see later changes. + + .. versionadded:: 0.80.0 + """ + return RemoteDandisetPath(parts=(), dandiset=self) / path + def get_asset(self, asset_id: str) -> RemoteAsset: """ Fetch the asset in this version of the Dandiset with the given asset @@ -2350,3 +2364,120 @@ class ZarrEntryServerData(BaseModel): last_modified: datetime = Field(alias="LastModified") etag: str = Field(alias="ETag") size: int = Field(alias="Size") + + +@dataclass +class RemoteDandisetPath(BasePath): + """A cached view of an asset or virtual directory in a Dandiset version. + + Zarr assets are leaves, just like blob assets. Their internal chunks are not + Dandiset children. Retrieving a full asset with `get_asset` makes a separate + request because the directory endpoint only supplies its identifier and URL. + + .. versionadded:: 0.80.0 + """ + + #: The Dandiset version containing this path. + dandiset: RemoteDandiset + _entry: dict[str, Any] | None = field(default=None, repr=False, compare=False) + _children: list[RemoteDandisetPath] | None = field( + default=None, repr=False, compare=False + ) + _missing: bool = field(default=False, repr=False, compare=False) + + def _get_subpath(self, name: str) -> RemoteDandisetPath: + if not name or "/" in name: + raise ValueError(f"Invalid path component: {name!r}") + if name == ".": + return self + if name == "..": + return self.parent + return type(self)(parts=(*self.parts, name), dandiset=self.dandiset) + + @property + def parent(self) -> RemoteDandisetPath: + return type(self)(parts=self.parts[:-1], dandiset=self.dandiset) + + def _list_entries(self, prefix: str) -> Iterator[dict[str, Any]]: + yield from self.dandiset.client.paginate( + f"{self.dandiset.version_api_path}assets/paths/", + params={"path_prefix": prefix}, + ) + + def _resolve(self) -> dict[str, Any] | None: + if self._entry is None and not self._missing: + try: + if self.is_root(): + self._load_children() + self._entry = { + "asset": None, + "aggregate_files": sum( + c.aggregate_files for c in self._children or [] + ), + "aggregate_size": sum(c.size for c in self._children or []), + } + else: + self._entry = next( + ( + e + for e in self._list_entries(str(self.parent)) + if e["path"] == str(self) + ), + None, + ) + except HTTP404Error: + self._missing = True + if self._entry is None: + self._missing = True + return self._entry + + def _require_entry(self) -> dict[str, Any]: + entry = self._resolve() + if entry is None: + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + return entry + + def exists(self) -> bool: + return self._resolve() is not None + + def is_file(self) -> bool: + entry = self._resolve() + return entry is not None and entry["asset"] is not None + + def is_dir(self) -> bool: + entry = self._resolve() + return entry is not None and entry["asset"] is None + + def _load_children(self) -> None: + if self._children is None: + self._children = [ + type(self)( + parts=tuple(entry["path"].split("/")), + dandiset=self.dandiset, + _entry=entry, + ) + for entry in self._list_entries(str(self)) + ] + + def iterdir(self) -> Iterator[RemoteDandisetPath]: + if self._require_entry()["asset"] is not None: + raise NotADirectoryError(str(self)) + self._load_children() + yield from self._children or [] + + @property + def aggregate_files(self) -> int: + """The recursive number of assets under this path (one for an asset).""" + return int(self._require_entry()["aggregate_files"]) + + @property + def size(self) -> int: + """The recursive size in bytes, as reported by the Archive.""" + return int(self._require_entry()["aggregate_size"]) + + def get_asset(self) -> RemoteAsset: + """Fetch the full asset record; directories raise IsADirectoryError.""" + asset = self._require_entry()["asset"] + if asset is None: + raise IsADirectoryError(str(self)) + return self.dandiset.get_asset(asset["asset_id"]) diff --git a/dandi/dandiset.py b/dandi/dandiset.py index 22cbce892..f600a326f 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -1,4 +1,5 @@ """Classes/utilities for support of a dandiset""" + from __future__ import annotations from collections.abc import Iterable, Iterator @@ -11,7 +12,9 @@ from . import get_logger from .consts import dandiset_metadata_file +from .exceptions import NotFoundError from .files import DandisetMetadataFile, LocalAsset, dandi_file, find_dandi_files +from .misctypes import BasePath from .utils import find_parent_directory_containing, under_paths, yaml_dump, yaml_load if TYPE_CHECKING: @@ -168,6 +171,18 @@ def assets(self, allow_all: bool = False) -> AssetView: data[PurePosixPath(df.path)] = df return AssetView(data) + def get_path(self, path: str = "") -> LocalDandisetPath: + """Browse a snapshot of the Dandiset's discoverable assets. + + Discovery runs once for the whole Dandiset and includes generic assets. + Empty and hidden directories are excluded by normal discovery rules; + Zarr directories are represented as single assets. Descendants share + the snapshot. Call this method again to refresh it. + + .. versionadded:: 0.80.0 + """ + return LocalDandisetPath(parts=(), assets=self.assets(allow_all=True)) / path + def metadata_file(self) -> DandisetMetadataFile: df = dandi_file(self._metadata_file_obj, dandiset_path=self.path) assert isinstance(df, DandisetMetadataFile) @@ -192,3 +207,70 @@ def under_paths(self, paths: Iterable[str | PurePath]) -> Iterator[LocalAsset]: # contain '.' or '..' for p in under_paths(self.data.keys(), paths): yield self.data[p] + + +@dataclass +class LocalDandisetPath(BasePath): + """An asset or directory in a local Dandiset discovery snapshot. + + .. versionadded:: 0.80.0 + """ + + #: Shared discovery results, including generic assets. + assets: AssetView + + def _get_subpath(self, name: str) -> LocalDandisetPath: + if not name or "/" in name: + raise ValueError(f"Invalid path component: {name!r}") + if name == ".": + return self + if name == "..": + return self.parent + return type(self)(parts=(*self.parts, name), assets=self.assets) + + @property + def parent(self) -> LocalDandisetPath: + return type(self)(parts=self.parts[:-1], assets=self.assets) + + def _descendants(self) -> Iterator[LocalAsset]: + yield from self.assets.under_paths([PurePosixPath(str(self))]) + + def exists(self) -> bool: + return self.is_root() or next(self._descendants(), None) is not None + + def is_file(self) -> bool: + return PurePosixPath(str(self)) in self.assets.data + + def is_dir(self) -> bool: + return self.exists() and not self.is_file() + + def iterdir(self) -> Iterator[LocalDandisetPath]: + if not self.exists(): + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + if self.is_file(): + raise NotADirectoryError(str(self)) + names = {a.path.split("/")[len(self.parts)] for a in self._descendants()} + for name in sorted(names): + yield self / name + + @property + def aggregate_files(self) -> int: + """The recursive number of discoverable assets.""" + if not self.exists(): + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + return sum(1 for _ in self._descendants()) + + @property + def size(self) -> int: + """The total size in bytes of the assets below this path.""" + if not self.exists(): + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + return sum(a.size for a in self._descendants()) + + def get_asset(self) -> LocalAsset: + """Return the discovered asset; directories raise IsADirectoryError.""" + if not self.exists(): + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + if not self.is_file(): + raise IsADirectoryError(str(self)) + return self.assets.data[PurePosixPath(str(self))] diff --git a/dandi/tests/test_dandiset_paths.py b/dandi/tests/test_dandiset_paths.py new file mode 100644 index 000000000..748d09dae --- /dev/null +++ b/dandi/tests/test_dandiset_paths.py @@ -0,0 +1,283 @@ +from collections.abc import Callable +from pathlib import Path +from typing import Any +from urllib.parse import parse_qsl, urlsplit + +import pytest +import responses + +from .fixtures import SampleDandiset +from .test_files import mkpaths +from ..consts import DandiInstance +from ..dandiapi import DandiAPIClient, RemoteDandiset +from ..dandiset import Dandiset +from ..exceptions import NotFoundError + + +@pytest.mark.ai_generated +def test_local_path_tree(tmp_path: Path) -> None: + mkpaths( + tmp_path, + "dandiset.yaml", + "file.txt", + "sub-01/a.nwb", + "sub-01/b.nwb", + "record.zarr/chunk", + ".hidden/a.txt", + "empty/", + ) + (tmp_path / "file.txt").write_bytes(b"123") + root = Dandiset(tmp_path).get_path() + assert root.exists() and root.is_dir() and not root.is_file() + assert [p.name for p in root.iterdir()] == ["file.txt", "record.zarr", "sub-01"] + assert root.aggregate_files == 4 + assert root.size == 3 + assert root.parent == root + assert root / "." == root + assert (root / "sub-01/..") == root + assert (root / "sub-01").aggregate_files == 2 + assert [str(p) for p in (root / "sub-01").iterdir()] == [ + "sub-01/a.nwb", + "sub-01/b.nwb", + ] + assert (root / "file.txt").get_asset().path == "file.txt" + assert (root / "record.zarr").is_file() + assert not (root / "record.zarr/chunk").exists() + assert not (root / ".hidden").exists() + assert not (root / "empty").exists() + with pytest.raises(NotADirectoryError): + list((root / "record.zarr").iterdir()) + with pytest.raises(IsADirectoryError): + root.get_asset() + missing = root / "missing" + assert not missing.exists() and not missing.is_file() and not missing.is_dir() + for operation in ( + lambda: list(missing.iterdir()), + lambda: missing.size, + lambda: missing.aggregate_files, + missing.get_asset, + ): + with pytest.raises(NotFoundError): + operation() + with pytest.raises(ValueError, match="Absolute"): + root.joinpath("/etc") + with pytest.raises(ValueError): + root._get_subpath("") + with pytest.raises(ValueError): + root._get_subpath("a/b") + + +@pytest.mark.ai_generated +def test_local_empty_and_snapshot(tmp_path: Path) -> None: + mkpaths(tmp_path, "dandiset.yaml") + ds = Dandiset(tmp_path) + root = ds.get_path() + assert list(root.iterdir()) == [] + assert root.aggregate_files == root.size == 0 + mkpaths(tmp_path, "new.txt") + assert not (root / "new.txt").exists() + assert ds.get_path("new.txt").exists() + + +@pytest.mark.ai_generated +def test_local_symlink_directory(tmp_path: Path) -> None: + mkpaths(tmp_path, "dandiset.yaml", "target/a.txt") + try: + (tmp_path / "linked").symlink_to(tmp_path / "target", target_is_directory=True) + except OSError as exc: + pytest.skip(f"Cannot create directory symlink: {exc}") + assert not Dandiset(tmp_path).get_path("linked").exists() + + +def _entry(path: str, count: int, size: int, asset: Any = None) -> dict: + return { + "path": path, + "aggregate_files": count, + "aggregate_size": size, + "asset": asset, + } + + +def _query_matcher(expected: dict[str, str]) -> Callable[[Any], tuple[bool, str]]: + """Match the raw query so empty values work with all supported responses versions.""" + + def match(request: Any) -> tuple[bool, str]: + actual = dict(parse_qsl(urlsplit(request.url).query, keep_blank_values=True)) + valid = actual == expected + return valid, f"Query parameters do not match: {actual!r} != {expected!r}" + + return match + + +@pytest.mark.ai_generated +@responses.activate +def test_remote_listing_pagination_and_cached_children( + monkeypatch: pytest.MonkeyPatch, +) -> None: + url = "https://example.test/api/dandisets/000001/versions/draft/assets/paths/" + first = _entry( + "file.txt", 1, 5, {"asset_id": "test-id", "url": "https://example.test/blob"} + ) + directory = _entry("sub-01", 2, 9) + responses.get( + url, + json={"count": 2, "results": [first], "next": url + "?path_prefix=&page=2"}, + match=[_query_matcher({"path_prefix": ""})], + ) + responses.get( + url, + json={"count": 2, "results": [directory], "next": None}, + match=[_query_matcher({"path_prefix": "", "page": "2"})], + ) + responses.get( + url, + json={ + "count": 1, + "results": [_entry("sub-01/a.nwb", 1, 9, {"asset_id": "other"})], + "next": None, + }, + match=[_query_matcher({"path_prefix": "sub-01"})], + ) + monkeypatch.setenv("DANDI_PAGINATION_DISABLE_FALLBACK", "1") + with DandiAPIClient( + dandi_instance=DandiInstance( + name="test", gui="https://example.test", api="https://example.test/api/" + ) + ) as client: + root = RemoteDandiset( + client=client, identifier="000001", version="draft" + ).get_path() + responses.calls.reset() + children = list(root.iterdir()) + assert [str(p) for p in children] == ["file.txt", "sub-01"] + count = len(responses.calls) + assert count == 2 + assert children[0].is_file() and not children[0].is_dir() + assert children[1].is_dir() and not children[1].is_file() + assert children[0].size == 5 + assert children[1].aggregate_files == 2 + assert root.size == 14 and root.aggregate_files == 3 + assert list(root.iterdir()) == children + assert len(responses.calls) == count + nested = list(children[1].iterdir()) + assert nested[0].name == "a.nwb" + assert str(nested[0]) == "sub-01/a.nwb" + assert nested[0].size == 9 + assert len(responses.calls) == count + 1 + with pytest.raises(NotADirectoryError): + list(children[0].iterdir()) + with pytest.raises(IsADirectoryError): + children[1].get_asset() + assert root / "." == root and root / ".." == root + with pytest.raises(ValueError): + root._get_subpath("a/b") + with pytest.raises(ValueError): + root._get_subpath("") + + +@pytest.mark.ai_generated +@responses.activate +@pytest.mark.parametrize("status", [200, 404, 403]) +def test_remote_unlisted_path_errors(status: int) -> None: + import requests + + url = "https://example.test/api/dandisets/000001/versions/draft/assets/paths/" + responses.get(url, json={"results": [], "next": None}, status=status) + with DandiAPIClient( + dandi_instance=DandiInstance( + name="test", gui="https://example.test", api="https://example.test/api/" + ) + ) as client: + path = RemoteDandiset( + client=client, identifier="000001", version="draft" + ).get_path("missing") + if status == 403: + with pytest.raises(requests.HTTPError): + path.exists() + else: + assert not path.exists() and not path.is_file() and not path.is_dir() + with pytest.raises(NotFoundError): + path.get_asset() + with pytest.raises(NotFoundError): + list(path.iterdir()) + + +@pytest.mark.ai_generated +def test_path_listing_local_remote_parity(text_dandiset: SampleDandiset) -> None: + mkpaths(text_dandiset.dspath, "sub-01/session/a.txt", "sub-02/b.txt") + (text_dandiset.dspath / "sub-01/session/a.txt").write_bytes(b"alpha\n") + (text_dandiset.dspath / "sub-02/b.txt").write_bytes(b"beta\n") + text_dandiset.upload() + local = Dandiset(text_dandiset.dspath).get_path() + remote = text_dandiset.dandiset.get_path() + + def describe(root: Any) -> dict: + return { + str(p): (p.is_file(), p.size, p.aggregate_files) for p in root.iterdir() + } + + assert describe(local) == describe(remote) + assert local.size == remote.size + assert local.aggregate_files == remote.aggregate_files == 6 + assert describe(local / "subdir2") == describe(remote / "subdir2") + asset = (remote / "file.txt").get_asset() + assert asset.path == "file.txt" + assert asset.size == (local / "file.txt").size + assert not (remote / "absent").exists() + + +@pytest.mark.ai_generated +@responses.activate +@pytest.mark.parametrize("kind", ["blob", "zarr"]) +def test_remote_unlisted_asset_can_be_fetched(kind: str) -> None: + base = "https://example.test/api/dandisets/000001/versions/draft/assets/" + name = "sample.zarr" if kind == "zarr" else "file.txt" + responses.get( + base + "paths/", + json={"results": [_entry(name, 1, 5, {"asset_id": "id"})], "next": None}, + ) + responses.get( + base + "id/info/", + json={ + "asset_id": "id", + "path": name, + "size": 5, + kind: "storage-id", + "created": "2026-01-01T00:00:00Z", + "modified": "2026-01-01T00:00:00Z", + }, + ) + with DandiAPIClient( + dandi_instance=DandiInstance( + name="test", gui="https://example.test", api="https://example.test/api/" + ) + ) as client: + path = RemoteDandiset(client, "000001", "draft").get_path(name) + assert path.exists() and path.is_file() and not path.is_dir() + assert path.size == 5 and path.aggregate_files == 1 + asset = path.get_asset() + assert asset.path == name and asset.size == 5 + + +@pytest.mark.ai_generated +@responses.activate +@pytest.mark.parametrize("status", [200, 404]) +def test_remote_empty_or_missing_root(status: int) -> None: + responses.get( + "https://example.test/api/dandisets/000001/versions/draft/assets/paths/", + json={"results": [], "next": None}, + status=status, + ) + with DandiAPIClient( + dandi_instance=DandiInstance( + name="test", gui="https://example.test", api="https://example.test/api/" + ) + ) as client: + root = RemoteDandiset(client, "000001", "draft").get_path() + assert root.exists() == (status == 200) + if status == 200: + assert list(root.iterdir()) == [] + assert root.size == root.aggregate_files == 0 + else: + with pytest.raises(NotFoundError): + list(root.iterdir()) diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 8212711d0..bb8b9439e 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -49,6 +49,40 @@ Dandisets .. autoclass:: RemoteDandiset() +Browsing directories +^^^^^^^^^^^^^^^^^^^^ + +Use ``RemoteDandiset.get_path()`` to list one level without retrieving every asset: + +.. code-block:: python + + with DandiAPIClient() as client: + root = client.get_dandiset("000026", "draft").get_path() + for entry in root.iterdir(): + print(entry.name, entry.is_dir(), entry.aggregate_files, entry.size) + +The result follows the ``BasePath`` interface: ``/``, ``parent``, ``iterdir()``, +``exists()``, ``is_file()``, ``is_dir()`` and ``size``. Both blob and Zarr assets +are files in this tree; Zarr chunks are not children. Call ``entry.get_asset()`` +to retrieve the full asset record. + +Remote listing costs one paginated request sequence per directory. Listed +children already contain recursive sizes and counts, so inspecting those +properties does not fetch each asset. Resolving an arbitrary unlisted path +requires listing its parent. A full asset record requires an additional request. +Listings are cached on the path objects; create a new root to see later changes. +Authorization and server errors propagate to the caller. + +For a local Dandiset, ``Dandiset(directory).get_path()`` provides the same path +operations. It discovers all assets once using DANDI's existing discovery rules, +including generic files. Empty directories, dot-prefixed paths and directory +symlinks follow those rules; they are not additional assets. Metadata in +``dandiset.yaml`` is not part of the asset tree. Local sizes are calculated from +the files; remote sizes come from Archive aggregates. + +.. autoclass:: RemoteDandisetPath() + :show-inheritance: + .. autoclass:: Version() :inherited-members: BaseModel :exclude-members: Config, JSON_EXCLUDE