From 2de9155c13061372cf8e9c77f0372e074f3de885 Mon Sep 17 00:00:00 2001 From: Bhaskar Gurram Date: Fri, 9 Oct 2026 14:38:02 +0000 Subject: [PATCH 1/2] feat(ogc): accept date, datetime and Timestamp values in date arguments A datetime.date, datetime.datetime or pandas.Timestamp passed to a date argument (time=[df.index.min(), None]) failed inside the formatter with AttributeError ('endswith') or, for a lone datetime, TypeError (not iterable), naming no argument. Read them like the equivalent string: aware values convert to UTC, naive ones use the local zone as naive strings do, a date is midnight of that day, and NaT is an open bound. Any other type raises ValueError naming the argument. Follow-up named in #441. --- NEWS.md | 2 + dataretrieval/ogc/dates.py | 70 +++++++++++++++++++++--------- tests/waterdata_test.py | 12 ++++++ tests/waterdata_utils_test.py | 80 +++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 19 deletions(-) diff --git a/NEWS.md b/NEWS.md index f683c9d2..2933de8c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**10/09/2026:** Date arguments of the OGC getters (`get_daily()`, `get_continuous()`, `waterdata.get_ratings()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument) accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. They are read like the equivalent string: an aware value is converted to UTC, a naive one is read in the local time zone as a naive string is, a `date` is midnight of that day, and `NaT` is an open bound like `None`. **Bug fix:** these values used to fail inside the package with `AttributeError: ... has no attribute 'endswith'`, or `TypeError: ... is not iterable` for a lone `datetime`, which named no argument. Any other type, such as a number, now raises `ValueError` naming the argument and the types it accepts. + **10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter. **10/07/2026:** **Bug fix:** a `".."` endpoint in a two-value date range -- `time=["2024-01-01", ".."]`, the open-ended form shown in the `waterdata.get_ratings()` docstring -- discarded the whole range, because `..` was not recognized as an open bound and failed to parse as a date. `waterdata.get_ratings()` then searched with no `datetime` at all and returned every rating regardless of date (59 instead of 44 for the docstring's bounding box from 2026-09-20 on), and the OGC getters (`get_daily()`, `get_continuous()`, `get_field_measurements()`, and the other `time`, `begin`, `end`, and `last_modified` arguments) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format". `".."` is now an open bound like `None`, so `["2024-01-01", ".."]` sends the same range as `"2024-01-01/.."` and `["2024-01-01", None]`. diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 0fe5cd9e..319df8be 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -8,7 +8,8 @@ from __future__ import annotations import re -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence +from datetime import date as _date from datetime import datetime from zoneinfo import ZoneInfo @@ -65,7 +66,13 @@ def _parse_datetime(value: str) -> datetime | None: _OPEN_BOUND = ".." -def _is_blank(dt: str | None) -> bool: +# One element of a date argument: an ISO 8601 string, or a ``date``, +# ``datetime`` or ``pandas.Timestamp`` (both subclasses of ``datetime.date``), +# with ``None`` (or ``NaN`` / ``NaT``) for an open bound. +_DateLike = str | _date | None + + +def _is_blank(dt: _DateLike) -> bool: """True for a None, NaN, empty-string, or ``..`` element. Each is a spelling of an open bound, so ``["2024-01-01", ".."]`` means the @@ -74,22 +81,42 @@ def _is_blank(dt: str | None) -> bool: return dt is None or bool(pd.isna(dt)) or dt in ("", _OPEN_BOUND) -def _format_one(dt: str | None, *, date: bool, name: str) -> str: - """Format a single datetime element for inclusion in the API time arg. +def _to_datetime(dt: str | _date, *, name: str) -> datetime: + """Read one non-blank element as a ``datetime`` (naive iff it has no zone). - Raises ``ValueError`` naming *name* when the element is not blank and - matches no supported format. + A ``date`` is midnight of that day, like the string ``"2024-01-01"``. + Raises ``ValueError`` naming *name* for an unreadable string or a value of + another type. """ - if dt is None or _is_blank(dt): - return _OPEN_BOUND - parsed = _parse_datetime(dt) + if isinstance(dt, pd.Timestamp): + # A plain ``datetime``: ``Timestamp.astimezone()`` needs an explicit zone. + converted: datetime = dt.to_pydatetime(warn=False) + return converted + if isinstance(dt, datetime): + return dt + if isinstance(dt, _date): + return datetime(dt.year, dt.month, dt.day) # noqa: DTZ001 + parsed = _parse_datetime(dt) if isinstance(dt, str) else None if parsed is None: raise ValueError( f"{name} could not be read as a date or datetime: {dt!r}. " "Pass an ISO 8601 date or datetime such as '2024-01-01' or " - "'2024-01-01T12:00:00Z', and None for an open end of a range " + "'2024-01-01T12:00:00Z' (a datetime.date, datetime.datetime or " + "pandas.Timestamp also works), and None for an open end of a range " "(['2024-01-01', None])." ) + return parsed + + +def _format_one(dt: _DateLike, *, date: bool, name: str) -> str: + """Format a single datetime element for inclusion in the API time arg. + + Raises ``ValueError`` naming *name* when the element is not blank and + cannot be read as a date or datetime. + """ + if dt is None or _is_blank(dt): + return _OPEN_BOUND + parsed = _to_datetime(dt, name=name) if date: return parsed.strftime("%Y-%m-%d") # Naive inputs are interpreted in the system local zone (for backwards @@ -101,11 +128,14 @@ def _format_one(dt: str | None, *, date: bool, name: str) -> str: def _coerce_to_list( - datetime_input: str | Sequence[str | None], + datetime_input: str | _date | Sequence[_DateLike], name: str = "date input", -) -> list[str | None]: +) -> list[_DateLike]: """Normalize datetime input to a list, raising on invalid shapes.""" - if isinstance(datetime_input, str): + if isinstance(datetime_input, (str, _date)) or not isinstance( + datetime_input, Iterable + ): + # A lone value; anything that is not a date is rejected per element. return [datetime_input] if isinstance(datetime_input, Mapping): raise TypeError( @@ -120,13 +150,13 @@ def _is_passthrough(single: str) -> bool: return bool(_DURATION_RE.match(single) or "/" in single) -def _all_blank(items: list[str | None]) -> bool: +def _all_blank(items: list[_DateLike]) -> bool: """True when every element is None, NaN, the empty string, or ``..``.""" return all(_is_blank(dt) for dt in items) def _format_api_dates( - datetime_input: str | Sequence[str | None] | None, + datetime_input: str | _date | Sequence[_DateLike] | None, date: bool = False, *, name: str = "date input", @@ -140,9 +170,11 @@ def _format_api_dates( Parameters ---------- - datetime_input : Union[str, List[Optional[str]], None] - A single date/datetime string or a list of one or two date/datetime - strings. Accepts formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or + datetime_input : Union[str, date, List[Optional[Union[str, date]]], None] + A single date/datetime or a list of one or two of them. Each may be a + ``datetime.date``, ``datetime.datetime`` or ``pandas.Timestamp`` (a + naive one is read in the local time zone, as a naive string is), or a + string. Strings accept formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or without ``Z``/numeric offset), or relative periods (e.g., "P7D" / "PT36H"). Range endpoints may be ``None``/``NaN``/empty or ``".."`` to denote a half-bounded range. @@ -174,7 +206,7 @@ def _format_api_dates( ------ ValueError If `datetime_input` contains more than two values, or an element that - is not blank matches no supported format. + is not blank matches no supported format or is of another type. Notes ----- diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 541f7a24..ae6ed660 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -619,6 +619,18 @@ def test_construct_api_requests_two_element_date_list_becomes_interval(): assert "time=2024-01-01%2F2024-01-31" in str(req.url) +def test_construct_api_requests_accepts_timestamp_bounds(): + """A ``pandas.Timestamp`` bound, as in ``time=[df.index.min(), None]``, + builds the same request as its string spelling instead of raising + ``AttributeError``.""" + req = _construct_api_requests( + "daily", + monitoring_location_id="USGS-05427718", + time=[pd.Timestamp("2024-01-01", tz="UTC"), None], + ) + assert "time=2024-01-01%2F.." in str(req.url) + + # --- mocked getter smoke tests ------------------------------------------------ # These replace what used to be ~34 live calls to the Water Data API. Each one # serves a committed fixture (``tests/data/waterdata_ogc_fixtures.json``, two diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 281744c0..157970e0 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1049,6 +1049,86 @@ def test_format_api_dates_rejects_mapping(): _format_api_dates({"2024-01-01": "ignored"}) +@pytest.mark.parametrize( + "value, date, expected", + [ + (datetime.date(2024, 1, 1), True, "2024-01-01"), + ([datetime.date(2024, 1, 1), None], True, "2024-01-01/.."), + ( + [datetime.date(2024, 1, 1), datetime.date(2024, 2, 1)], + True, + "2024-01-01/2024-02-01", + ), + (pd.Timestamp("2024-01-01T10:30:00", tz="UTC"), True, "2024-01-01"), + ( + pd.Timestamp("2024-01-01T10:30:00", tz="UTC"), + False, + "2024-01-01T10:30:00Z", + ), + ( + datetime.datetime( + 2024, 1, 1, 6, 0, tzinfo=datetime.timezone(datetime.timedelta(hours=-4)) + ), + False, + "2024-01-01T10:00:00Z", + ), + ( + [pd.Timestamp("2024-01-01", tz="UTC"), pd.NaT], + False, + "2024-01-01T00:00:00Z/..", + ), + ( + ["2024-01-01T00:00:00Z", pd.Timestamp("2024-02-01", tz="UTC")], + False, + "2024-01-01T00:00:00Z/2024-02-01T00:00:00Z", + ), + ], + ids=[ + "date", + "date_open_end", + "date_pair", + "aware_timestamp_date_only", + "aware_timestamp_to_utc", + "aware_datetime_offset_to_utc", + "nat_is_an_open_bound", + "mixed_string_and_timestamp", + ], +) +def test_format_api_dates_accepts_date_and_datetime_objects(value, date, expected): + """``datetime.date``, ``datetime.datetime`` and ``pandas.Timestamp`` used to + raise ``AttributeError`` (no ``endswith``) or ``TypeError`` (a lone datetime + is not iterable) from inside the formatter. They are read like the + equivalent string, and ``NaT`` is an open bound like ``None``.""" + assert _format_api_dates(value, date=date) == expected + + +@pytest.mark.parametrize("date", [True, False], ids=["date_only", "datetime"]) +def test_format_api_dates_reads_naive_objects_like_naive_strings(date): + """A naive ``datetime``/``Timestamp`` is local time, the same rule a naive + string follows, so both spellings of one instant build the same filter in + any time zone. A ``date`` is midnight of that day, like ``"2024-01-01"``.""" + as_string = _format_api_dates(["2024-01-01T10:00:00", None], date=date) + naive_datetime = [datetime.datetime(2024, 1, 1, 10), None] # noqa: DTZ001 + assert _format_api_dates(naive_datetime, date=date) == as_string + naive_timestamp = [pd.Timestamp("2024-01-01T10:00:00"), None] + assert _format_api_dates(naive_timestamp, date=date) == as_string + assert _format_api_dates(datetime.date(2024, 1, 1), date=date) == ( + _format_api_dates("2024-01-01", date=date) + ) + + +@pytest.mark.parametrize("value", [20240101, [20240101, None], 1.5]) +def test_format_api_dates_rejects_other_types_naming_the_argument(value): + """A number is not a date. It used to fail with an ``AttributeError`` from + inside the formatter; the message now names the caller's argument and the + types it accepts.""" + with pytest.raises(ValueError) as excinfo: + _format_api_dates(value, name="time") + message = str(excinfo.value) + assert message.startswith("time could not be read as a date") + assert "pandas.Timestamp" in message + + def _make_response(status, body, reason=None, content_type="text/html"): headers = {"Content-Type": content_type} extensions = {} From e2f83608ddbf95db6b2e2d088a2d8f11b424cd54 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Mon, 12 Oct 2026 10:19:28 -0500 Subject: [PATCH 2/2] fix(ratings): accept a lone date or Timestamp in get_ratings(time=...) _validate_time_no_duration iterated a lone Timestamp/date and raised TypeError before the formatter could read it; it now normalizes with _coerce_to_list and checks only string elements. Also: - drop the redundant date check in _coerce_to_list (a date is not iterable), and update the Mapping TypeError to name the accepted types; - read a bytes value as one element, so it gets the message naming the accepted types rather than "takes at most 2 values" over its byte codes; - get_ratings' time docstring names the accepted date types; - tighten the NEWS entry and date it by its land, above #446's. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HBt3Ee7eneKtdKQWq2T3YK --- NEWS.md | 2 +- dataretrieval/ogc/dates.py | 10 ++++++---- dataretrieval/waterdata/ratings.py | 9 ++++++--- tests/waterdata_ratings_test.py | 22 ++++++++++++++++++++++ tests/waterdata_utils_test.py | 8 +++++--- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index 889b29ac..a9c540f7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -**10/09/2026:** Date arguments of the OGC getters (`get_daily()`, `get_continuous()`, `waterdata.get_ratings()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument) accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. They are read like the equivalent string: an aware value is converted to UTC, a naive one is read in the local time zone as a naive string is, a `date` is midnight of that day, and `NaT` is an open bound like `None`. **Bug fix:** these values used to fail inside the package with `AttributeError: ... has no attribute 'endswith'`, or `TypeError: ... is not iterable` for a lone `datetime`, which named no argument. Any other type, such as a number, now raises `ValueError` naming the argument and the types it accepts. +**10/12/2026:** Date arguments of the `waterdata` and `ngwmn` OGC getters (`time`, `begin`, `end`, `datetime`, `last_modified`) and of `waterdata.get_ratings()` accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. Each is read like the equivalent string: a `date` is midnight, and `NaT` is an open bound. **Bug fix:** these values used to raise `AttributeError` or `TypeError` without naming the argument. Any other type, such as a number, now raises `ValueError` naming the argument and the accepted types. **10/11/2026:** **Bug fix:** a list passed to most `waterdata` OGC filters, such as `get_daily(approval_status=["Approved", "Provisional"])`, returned no rows: the Water Data API reads a comma-joined value as one value for all but a few identifier and code filters, and rejects a numeric list with HTTP 400. Lists are now sent as a CQL2 `in` filter. A list combined with `filter=` in `get_monitoring_locations()` and `get_combined_metadata()` is no longer ignored. **Bug fix:** `filter_lang="cql-json"` failed with HTTP 400 in every `waterdata` OGC getter. The filter is now sent as the request body; one that is not a JSON object raises `ValueError`. A blank `filter=""` is now ignored rather than failing with HTTP 400. `ngwmn` is unchanged. diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 319df8be..b0d1de61 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -89,7 +89,8 @@ def _to_datetime(dt: str | _date, *, name: str) -> datetime: another type. """ if isinstance(dt, pd.Timestamp): - # A plain ``datetime``: ``Timestamp.astimezone()`` needs an explicit zone. + # ``Timestamp.astimezone`` is ``tz_convert`` and requires a zone, so + # hand ``_format_one`` a plain ``datetime``. converted: datetime = dt.to_pydatetime(warn=False) return converted if isinstance(dt, datetime): @@ -132,14 +133,15 @@ def _coerce_to_list( name: str = "date input", ) -> list[_DateLike]: """Normalize datetime input to a list, raising on invalid shapes.""" - if isinstance(datetime_input, (str, _date)) or not isinstance( + if isinstance(datetime_input, (str, bytes)) or not isinstance( datetime_input, Iterable ): - # A lone value; anything that is not a date is rejected per element. + # A lone value (a ``date`` is not iterable; ``bytes`` would iterate as + # integers); a non-date is rejected per element. return [datetime_input] if isinstance(datetime_input, Mapping): raise TypeError( - f"{name} must be a string or sequence of strings, " + f"{name} must be a date, a string, or a sequence of them, " f"not {type(datetime_input).__name__}." ) return list(datetime_input) diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index f1e4ab8b..b1b7fb51 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -20,7 +20,7 @@ from dataretrieval._validation import render_options from dataretrieval.exceptions import DataRetrievalError, SkippedRatingWarning -from dataretrieval.ogc.dates import _DURATION_RE, _format_api_dates +from dataretrieval.ogc.dates import _DURATION_RE, _coerce_to_list, _format_api_dates from dataretrieval.ogc.errors import _raise_for_non_200 from dataretrieval.ogc.filters import _cql_in_clause from dataretrieval.ogc.requests import _check_monitoring_location_id @@ -81,7 +81,7 @@ def get_ratings( (the default), the parsed ``DataFrame`` is returned without persisting the bytes to disk; ``df.attrs["url"]`` still records where each rating came from. - time : string or list of strings, optional + time : string, date, datetime, or pandas.Timestamp, or a list of them, optional STAC ``datetime`` filter (passed through verbatim under that name) — a single date / datetime, an interval (``"start/end"``, optionally half-bounded with ``..``), or a list of two values @@ -221,7 +221,10 @@ def _validate_time_no_duration(time: str | list[str] | None) -> None: """Raise ValueError if ``time`` contains an ISO 8601 duration.""" if time is None: return - if any(_DURATION_RE.match(str(v)) for v in _as_list(time)): + # ``_coerce_to_list`` keeps a lone ``date``/``Timestamp`` whole, where + # ``_as_list`` would try to iterate it. + values = _coerce_to_list(time, "time") + if any(isinstance(v, str) and _DURATION_RE.match(v) for v in values): raise ValueError( "ISO 8601 durations (e.g. 'P7D') are not supported in `time` " "for the rating-curve service. Provide a date or interval instead." diff --git a/tests/waterdata_ratings_test.py b/tests/waterdata_ratings_test.py index e85f9edc..cf225e76 100644 --- a/tests/waterdata_ratings_test.py +++ b/tests/waterdata_ratings_test.py @@ -192,6 +192,28 @@ def test_get_ratings_keeps_a_dotdot_open_bound_in_time(httpx_mock): assert params["datetime"] == ["2026-04-29T00:00:00Z/.."] +@pytest.mark.parametrize( + "time", + [ + pd.Timestamp("2026-04-29", tz="UTC"), + [pd.Timestamp("2026-04-29", tz="UTC"), None], + ], + ids=["lone", "range"], +) +def test_get_ratings_accepts_a_timestamp_time(httpx_mock, time): + """The duration check iterated a lone ``Timestamp`` and raised + ``TypeError`` before the date formatter could read it.""" + httpx_mock.add_response( + method="GET", url=STAC_SEARCH_RE, json=_stub_search_response() + ) + get_ratings( + monitoring_location_id="USGS-01104475", time=time, download_and_parse=False + ) + (request,) = httpx_mock.get_requests() + datetime_param = parse_qs(urlsplit(str(request.url)).query)["datetime"][0] + assert datetime_param.startswith("2026-04-29T00:00:00Z") + + def test_get_ratings_rejects_an_unreadable_time_before_any_request(httpx_mock): """An unreadable bound used to drop the ``datetime`` filter, so the search silently returned every rating regardless of date.""" diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 157970e0..5ef495f7 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1037,7 +1037,7 @@ def test_format_api_dates_names_the_callers_parameter(): ``datetime_input`` -- a local of this private helper that no public getter accepts, so correcting the argument the message named sent an unrecognized parameter.""" - with pytest.raises(TypeError, match="^time must be a string"): + with pytest.raises(TypeError, match="^time must be a date, a string"): _format_api_dates({"2024-01-01": "ignored"}, name="time") @@ -1045,7 +1045,9 @@ def test_format_api_dates_rejects_mapping(): """`time={"2024-01-01": "x"}` would materialize as the keys list, accepting input the user clearly didn't intend. """ - with pytest.raises(TypeError, match="date input must be a string or sequence"): + with pytest.raises( + TypeError, match="date input must be a date, a string, or a sequence" + ): _format_api_dates({"2024-01-01": "ignored"}) @@ -1117,7 +1119,7 @@ def test_format_api_dates_reads_naive_objects_like_naive_strings(date): ) -@pytest.mark.parametrize("value", [20240101, [20240101, None], 1.5]) +@pytest.mark.parametrize("value", [20240101, [20240101, None], 1.5, b"2024-01-01"]) def test_format_api_dates_rejects_other_types_naming_the_argument(value): """A number is not a date. It used to fail with an ``AttributeError`` from inside the formatter; the message now names the caller's argument and the