Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
**10/09/2026:** Date arguments of the `waterdata` and `ngwmn` OGC getters (`time`, `begin`, `end`, `datetime`, `last_modified`) and of `waterdata.get_ratings()` accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. Each is read like the equivalent string: an aware value is converted to UTC, a naive one is read in local time, a `date` is midnight, and `NaT` is an open bound. **Bug fix:** these values used to raise `AttributeError` or `TypeError` without naming the argument. Any other type, such as a number, now raises `ValueError` naming the argument and the accepted types.

**10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter.

**10/07/2026:** **Bug fix:** a `".."` endpoint in a two-value date range -- `time=["2024-01-01", ".."]`, the open-ended form shown in the `waterdata.get_ratings()` docstring -- discarded the whole range, because `..` was not recognized as an open bound and failed to parse as a date. `waterdata.get_ratings()` then searched with no `datetime` at all and returned every rating regardless of date (59 instead of 44 for the docstring's bounding box from 2026-09-20 on), and the OGC getters (`get_daily()`, `get_continuous()`, `get_field_measurements()`, and the other `time`, `begin`, `end`, and `last_modified` arguments) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format". `".."` is now an open bound like `None`, so `["2024-01-01", ".."]` sends the same range as `"2024-01-01/.."` and `["2024-01-01", None]`.
Expand Down
72 changes: 52 additions & 20 deletions dataretrieval/ogc/dates.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
from __future__ import annotations

import re
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from datetime import date as _date
from datetime import datetime
from zoneinfo import ZoneInfo

Expand Down Expand Up @@ -65,7 +66,13 @@ def _parse_datetime(value: str) -> datetime | None:
_OPEN_BOUND = ".."


def _is_blank(dt: str | None) -> bool:
# One element of a date argument: an ISO 8601 string, or a ``date``,
# ``datetime`` or ``pandas.Timestamp`` (both subclasses of ``datetime.date``),
# with ``None`` (or ``NaN`` / ``NaT``) for an open bound.
_DateLike = str | _date | None


def _is_blank(dt: _DateLike) -> bool:
"""True for a None, NaN, empty-string, or ``..`` element.

Each is a spelling of an open bound, so ``["2024-01-01", ".."]`` means the
Expand All @@ -74,22 +81,43 @@ def _is_blank(dt: str | None) -> bool:
return dt is None or bool(pd.isna(dt)) or dt in ("", _OPEN_BOUND)


def _format_one(dt: str | None, *, date: bool, name: str) -> str:
"""Format a single datetime element for inclusion in the API time arg.
def _to_datetime(dt: str | _date, *, name: str) -> datetime:
"""Read one non-blank element as a ``datetime`` (naive iff it has no zone).

Raises ``ValueError`` naming *name* when the element is not blank and
matches no supported format.
A ``date`` is midnight of that day, like the string ``"2024-01-01"``.
Raises ``ValueError`` naming *name* for an unreadable string or a value of
another type.
"""
if dt is None or _is_blank(dt):
return _OPEN_BOUND
parsed = _parse_datetime(dt)
if isinstance(dt, pd.Timestamp):
# ``Timestamp.astimezone`` is ``tz_convert`` and requires a zone, so
# hand ``_format_one`` a plain ``datetime``.
converted: datetime = dt.to_pydatetime(warn=False)
return converted
if isinstance(dt, datetime):
return dt
if isinstance(dt, _date):
return datetime(dt.year, dt.month, dt.day) # noqa: DTZ001
parsed = _parse_datetime(dt) if isinstance(dt, str) else None
if parsed is None:
raise ValueError(
f"{name} could not be read as a date or datetime: {dt!r}. "
"Pass an ISO 8601 date or datetime such as '2024-01-01' or "
"'2024-01-01T12:00:00Z', and None for an open end of a range "
"'2024-01-01T12:00:00Z' (a datetime.date, datetime.datetime or "
"pandas.Timestamp also works), and None for an open end of a range "
"(['2024-01-01', None])."
)
return parsed


def _format_one(dt: _DateLike, *, date: bool, name: str) -> str:
"""Format a single datetime element for inclusion in the API time arg.

Raises ``ValueError`` naming *name* when the element is not blank and
cannot be read as a date or datetime.
"""
if dt is None or _is_blank(dt):
return _OPEN_BOUND
parsed = _to_datetime(dt, name=name)
if date:
return parsed.strftime("%Y-%m-%d")
# Naive inputs are interpreted in the system local zone (for backwards
Expand All @@ -101,15 +129,17 @@ def _format_one(dt: str | None, *, date: bool, name: str) -> str:


def _coerce_to_list(
datetime_input: str | Sequence[str | None],
datetime_input: str | _date | Sequence[_DateLike],
name: str = "date input",
) -> list[str | None]:
) -> list[_DateLike]:
"""Normalize datetime input to a list, raising on invalid shapes."""
if isinstance(datetime_input, str):
if isinstance(datetime_input, str) or not isinstance(datetime_input, Iterable):
# A lone value (a ``date`` is not iterable); a non-date is rejected
# per element.
return [datetime_input]
if isinstance(datetime_input, Mapping):
raise TypeError(
f"{name} must be a string or sequence of strings, "
f"{name} must be a date, a string, or a sequence of them, "
f"not {type(datetime_input).__name__}."
)
return list(datetime_input)
Expand All @@ -120,13 +150,13 @@ def _is_passthrough(single: str) -> bool:
return bool(_DURATION_RE.match(single) or "/" in single)


def _all_blank(items: list[str | None]) -> bool:
def _all_blank(items: list[_DateLike]) -> bool:
"""True when every element is None, NaN, the empty string, or ``..``."""
return all(_is_blank(dt) for dt in items)


def _format_api_dates(
datetime_input: str | Sequence[str | None] | None,
datetime_input: str | _date | Sequence[_DateLike] | None,
date: bool = False,
*,
name: str = "date input",
Expand All @@ -140,9 +170,11 @@ def _format_api_dates(

Parameters
----------
datetime_input : Union[str, List[Optional[str]], None]
A single date/datetime string or a list of one or two date/datetime
strings. Accepts formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or
datetime_input : Union[str, date, List[Optional[Union[str, date]]], None]
A single date/datetime or a list of one or two of them. Each may be a
``datetime.date``, ``datetime.datetime`` or ``pandas.Timestamp`` (a
naive one is read in the local time zone, as a naive string is), or a
string. Strings accept formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or
without ``Z``/numeric offset), or relative periods (e.g., "P7D" /
"PT36H"). Range endpoints may be ``None``/``NaN``/empty or ``".."``
to denote a half-bounded range.
Expand Down Expand Up @@ -174,7 +206,7 @@ def _format_api_dates(
------
ValueError
If `datetime_input` contains more than two values, or an element that
is not blank matches no supported format.
is not blank matches no supported format or is of another type.

Notes
-----
Expand Down
7 changes: 5 additions & 2 deletions dataretrieval/waterdata/ratings.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

from dataretrieval._validation import render_options
from dataretrieval.exceptions import DataRetrievalError, SkippedRatingWarning
from dataretrieval.ogc.dates import _DURATION_RE, _format_api_dates
from dataretrieval.ogc.dates import _DURATION_RE, _coerce_to_list, _format_api_dates
from dataretrieval.ogc.errors import _raise_for_non_200
from dataretrieval.ogc.filters import _quote_cql_str
from dataretrieval.ogc.requests import _check_monitoring_location_id
Expand Down Expand Up @@ -221,7 +221,10 @@ def _validate_time_no_duration(time: str | list[str] | None) -> None:
"""Raise ValueError if ``time`` contains an ISO 8601 duration."""
if time is None:
return
if any(_DURATION_RE.match(str(v)) for v in _as_list(time)):
# ``_coerce_to_list`` keeps a lone ``date``/``Timestamp`` whole, where
# ``_as_list`` would try to iterate it.
values = _coerce_to_list(time, "time")
if any(isinstance(v, str) and _DURATION_RE.match(v) for v in values):
raise ValueError(
"ISO 8601 durations (e.g. 'P7D') are not supported in `time` "
"for the rating-curve service. Provide a date or interval instead."
Expand Down
22 changes: 22 additions & 0 deletions tests/waterdata_ratings_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,28 @@ def test_get_ratings_keeps_a_dotdot_open_bound_in_time(httpx_mock):
assert params["datetime"] == ["2026-04-29T00:00:00Z/.."]


@pytest.mark.parametrize(
"time",
[
pd.Timestamp("2026-04-29", tz="UTC"),
[pd.Timestamp("2026-04-29", tz="UTC"), None],
],
ids=["lone", "range"],
)
def test_get_ratings_accepts_a_timestamp_time(httpx_mock, time):
"""The duration check iterated a lone ``Timestamp`` and raised
``TypeError`` before the date formatter could read it."""
httpx_mock.add_response(
method="GET", url=STAC_SEARCH_RE, json=_stub_search_response()
)
get_ratings(
monitoring_location_id="USGS-01104475", time=time, download_and_parse=False
)
(request,) = httpx_mock.get_requests()
datetime_param = parse_qs(urlsplit(str(request.url)).query)["datetime"][0]
assert datetime_param.startswith("2026-04-29T00:00:00Z")


def test_get_ratings_rejects_an_unreadable_time_before_any_request(httpx_mock):
"""An unreadable bound used to drop the ``datetime`` filter, so the search
silently returned every rating regardless of date."""
Expand Down
12 changes: 12 additions & 0 deletions tests/waterdata_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,18 @@ def test_construct_api_requests_two_element_date_list_becomes_interval():
assert "time=2024-01-01%2F2024-01-31" in str(req.url)


def test_construct_api_requests_accepts_timestamp_bounds():
"""A ``pandas.Timestamp`` bound, as in ``time=[df.index.min(), None]``,
builds the same request as its string spelling instead of raising
``AttributeError``."""
req = _construct_api_requests(
"daily",
monitoring_location_id="USGS-05427718",
time=[pd.Timestamp("2024-01-01", tz="UTC"), None],
)
assert "time=2024-01-01%2F.." in str(req.url)


# --- mocked getter smoke tests ------------------------------------------------
# These replace what used to be ~34 live calls to the Water Data API. Each one
# serves a committed fixture (``tests/data/waterdata_ogc_fixtures.json``, two
Expand Down
86 changes: 84 additions & 2 deletions tests/waterdata_utils_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1037,18 +1037,100 @@ def test_format_api_dates_names_the_callers_parameter():
``datetime_input`` -- a local of this private helper that no public getter
accepts, so correcting the argument the message named sent an
unrecognized parameter."""
with pytest.raises(TypeError, match="^time must be a string"):
with pytest.raises(TypeError, match="^time must be a date, a string"):
_format_api_dates({"2024-01-01": "ignored"}, name="time")


def test_format_api_dates_rejects_mapping():
"""`time={"2024-01-01": "x"}` would materialize as the keys list,
accepting input the user clearly didn't intend.
"""
with pytest.raises(TypeError, match="date input must be a string or sequence"):
with pytest.raises(
TypeError, match="date input must be a date, a string, or a sequence"
):
_format_api_dates({"2024-01-01": "ignored"})


@pytest.mark.parametrize(
"value, date, expected",
[
(datetime.date(2024, 1, 1), True, "2024-01-01"),
([datetime.date(2024, 1, 1), None], True, "2024-01-01/.."),
(
[datetime.date(2024, 1, 1), datetime.date(2024, 2, 1)],
True,
"2024-01-01/2024-02-01",
),
(pd.Timestamp("2024-01-01T10:30:00", tz="UTC"), True, "2024-01-01"),
(
pd.Timestamp("2024-01-01T10:30:00", tz="UTC"),
False,
"2024-01-01T10:30:00Z",
),
(
datetime.datetime(
2024, 1, 1, 6, 0, tzinfo=datetime.timezone(datetime.timedelta(hours=-4))
),
False,
"2024-01-01T10:00:00Z",
),
(
[pd.Timestamp("2024-01-01", tz="UTC"), pd.NaT],
False,
"2024-01-01T00:00:00Z/..",
),
(
["2024-01-01T00:00:00Z", pd.Timestamp("2024-02-01", tz="UTC")],
False,
"2024-01-01T00:00:00Z/2024-02-01T00:00:00Z",
),
],
ids=[
"date",
"date_open_end",
"date_pair",
"aware_timestamp_date_only",
"aware_timestamp_to_utc",
"aware_datetime_offset_to_utc",
"nat_is_an_open_bound",
"mixed_string_and_timestamp",
],
)
def test_format_api_dates_accepts_date_and_datetime_objects(value, date, expected):
"""``datetime.date``, ``datetime.datetime`` and ``pandas.Timestamp`` used to
raise ``AttributeError`` (no ``endswith``) or ``TypeError`` (a lone datetime
is not iterable) from inside the formatter. They are read like the
equivalent string, and ``NaT`` is an open bound like ``None``."""
assert _format_api_dates(value, date=date) == expected


@pytest.mark.parametrize("date", [True, False], ids=["date_only", "datetime"])
def test_format_api_dates_reads_naive_objects_like_naive_strings(date):
"""A naive ``datetime``/``Timestamp`` is local time, the same rule a naive
string follows, so both spellings of one instant build the same filter in
any time zone. A ``date`` is midnight of that day, like ``"2024-01-01"``."""
as_string = _format_api_dates(["2024-01-01T10:00:00", None], date=date)
naive_datetime = [datetime.datetime(2024, 1, 1, 10), None] # noqa: DTZ001
assert _format_api_dates(naive_datetime, date=date) == as_string
naive_timestamp = [pd.Timestamp("2024-01-01T10:00:00"), None]
assert _format_api_dates(naive_timestamp, date=date) == as_string
assert _format_api_dates(datetime.date(2024, 1, 1), date=date) == (
_format_api_dates("2024-01-01", date=date)
)


@pytest.mark.parametrize("value", [20240101, [20240101, None], 1.5])
def test_format_api_dates_rejects_other_types_naming_the_argument(value):
"""A number is not a date. It used to fail with an ``AttributeError`` from
inside the formatter; the message now names the caller's argument and the
types it accepts."""
with pytest.raises(ValueError) as excinfo:
_format_api_dates(value, name="time")
message = str(excinfo.value)
assert message.startswith("time could not be read as a date")
assert "pandas.Timestamp" in message


def _make_response(status, body, reason=None, content_type="text/html"):
headers = {"Content-Type": content_type}
extensions = {}
Expand Down
Loading