diff --git a/NEWS.md b/NEWS.md index 7ce6591d..f683c9d2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter. + **10/07/2026:** **Bug fix:** a `".."` endpoint in a two-value date range -- `time=["2024-01-01", ".."]`, the open-ended form shown in the `waterdata.get_ratings()` docstring -- discarded the whole range, because `..` was not recognized as an open bound and failed to parse as a date. `waterdata.get_ratings()` then searched with no `datetime` at all and returned every rating regardless of date (59 instead of 44 for the docstring's bounding box from 2026-09-20 on), and the OGC getters (`get_daily()`, `get_continuous()`, `get_field_measurements()`, and the other `time`, `begin`, `end`, and `last_modified` arguments) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format". `".."` is now an open bound like `None`, so `["2024-01-01", ".."]` sends the same range as `"2024-01-01/.."` and `["2024-01-01", None]`. **10/03/2026:** `waterdata.get_field_measurements()` and diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index ebbaec10..7fc351ba 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -265,7 +265,8 @@ def get_water_level( Vertical datum of the reported water level. datetime : str or iterable of str, optional Temporal filter — a single instant or a two-element ``[start, end]`` - range (ISO-8601 dates/datetimes); ``".."`` denotes an open end. + range (ISO-8601 dates/datetimes); ``None`` or ``".."`` denotes an + open end. properties : str or iterable of str, optional Subset of columns to return. ``None`` (default) returns all columns. limit : int, optional diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 33a70642..0fe5cd9e 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -74,13 +74,22 @@ def _is_blank(dt: str | None) -> bool: return dt is None or bool(pd.isna(dt)) or dt in ("", _OPEN_BOUND) -def _format_one(dt: str | None, *, date: bool) -> str | None: - """Format a single datetime element for inclusion in the API time arg.""" +def _format_one(dt: str | None, *, date: bool, name: str) -> str: + """Format a single datetime element for inclusion in the API time arg. + + Raises ``ValueError`` naming *name* when the element is not blank and + matches no supported format. + """ if dt is None or _is_blank(dt): return _OPEN_BOUND parsed = _parse_datetime(dt) if parsed is None: - return None + raise ValueError( + f"{name} could not be read as a date or datetime: {dt!r}. " + "Pass an ISO 8601 date or datetime such as '2024-01-01' or " + "'2024-01-01T12:00:00Z', and None for an open end of a range " + "(['2024-01-01', None])." + ) if date: return parsed.strftime("%Y-%m-%d") # Naive inputs are interpreted in the system local zone (for backwards @@ -155,24 +164,24 @@ def _format_api_dates( Returns ------- Union[str, None] - - If input is a single value, returns the formatted date/datetime string - or None if parsing fails. + - If input is a single value, returns the formatted date/datetime string. - If input is a list of two values, returns a date/datetime range string separated by "/" (e.g., "YYYY-MM-DD/YYYY-MM-DD" or "YYYY-MM-DDTHH:MM:SSZ/YYYY-MM-DDTHH:MM:SSZ"). - - Returns None if input is empty, all NA, or cannot be parsed. + - Returns None if input is None, empty, or every element is blank. Raises ------ ValueError - If `datetime_input` contains more than two values. + If `datetime_input` contains more than two values, or an element that + is not blank matches no supported format. Notes ----- - A single blank/NA value returns None. In a two-value range, a blank/NA or ``".."`` endpoint is rendered as ``".."`` to denote an open bound (e.g. ``"2024-01-01/.."``); the range is only None when *every* element - is blank/NA/``".."`` or any other element fails to parse. + is blank/NA/``".."``. - Supports ISO 8601 durations such as "P7D" and "PT36H" and pre-formatted intervals containing ``"/"``; both are passed through unchanged. - Converts datetimes to UTC and formats as ISO 8601 with 'Z' suffix when @@ -199,11 +208,4 @@ def _format_api_dates( if len(items) == 1 and isinstance(items[0], str) and _is_passthrough(items[0]): return items[0] - # Format each element; any element that fails to parse invalidates the range. - formatted: list[str] = [] - for dt in items: - one = _format_one(dt, date=date) - if one is None: - return None - formatted.append(one) - return "/".join(formatted) + return "/".join(_format_one(dt, date=date, name=name) for dt in items) diff --git a/dataretrieval/waterdata/measurements.py b/dataretrieval/waterdata/measurements.py index cfbcedeb..963128c9 100644 --- a/dataretrieval/waterdata/measurements.py +++ b/dataretrieval/waterdata/measurements.py @@ -115,6 +115,8 @@ def get_field_measurements( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -146,6 +148,8 @@ def get_field_measurements( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -438,6 +442,8 @@ def get_channel( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -491,6 +497,8 @@ def get_channel( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours diff --git a/dataretrieval/waterdata/metadata.py b/dataretrieval/waterdata/metadata.py index 9fbf1eaf..70664167 100644 --- a/dataretrieval/waterdata/metadata.py +++ b/dataretrieval/waterdata/metadata.py @@ -498,6 +498,8 @@ def get_time_series_metadata( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -516,6 +518,8 @@ def get_time_series_metadata( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -537,6 +541,8 @@ def get_time_series_metadata( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index b6698919..f78ea2fc 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -83,8 +83,10 @@ def get_ratings( where each rating came from. time : string or list of strings, optional STAC ``datetime`` filter (passed through verbatim under that name) - — a single date / datetime, or an interval (``"start/end"``, - optionally half-bounded with ``..``). ISO 8601 *durations* + — a single date / datetime, an interval (``"start/end"``, + optionally half-bounded with ``..``), or a list of two values + with ``None`` for an open end (``["2026-04-29", None]``). A value + that is not a date raises ``ValueError``. ISO 8601 *durations* (``"P1M"``, ``"PT36H"``, …) are **not** supported by the rating-curve service; passing one raises ``ValueError``. bbox : list of numbers, optional diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py index 8eec3aa4..96b2ff9f 100644 --- a/dataretrieval/waterdata/time_series.py +++ b/dataretrieval/waterdata/time_series.py @@ -131,6 +131,8 @@ def get_daily( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -152,6 +154,8 @@ def get_daily( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -374,6 +378,8 @@ def get_continuous( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -391,6 +397,8 @@ def get_continuous( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -572,6 +580,8 @@ def get_latest_continuous( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -593,6 +603,8 @@ def get_latest_continuous( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -787,6 +799,8 @@ def get_latest_daily( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours @@ -808,6 +822,8 @@ def get_latest_daily( * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" + * A list of two values: ["2018-02-12", "2018-03-18"], with None + for an open end: ["2018-02-12", None] * Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours diff --git a/tests/waterdata_ratings_test.py b/tests/waterdata_ratings_test.py index b81c1b06..f0c96967 100644 --- a/tests/waterdata_ratings_test.py +++ b/tests/waterdata_ratings_test.py @@ -190,6 +190,18 @@ def test_get_ratings_keeps_a_dotdot_open_bound_in_time(httpx_mock): assert params["datetime"] == ["2026-04-29T00:00:00Z/.."] +def test_get_ratings_rejects_an_unreadable_time_before_any_request(httpx_mock): + """An unreadable bound used to drop the ``datetime`` filter, so the search + silently returned every rating regardless of date.""" + with pytest.raises(ValueError, match=r"^time could not be read as a date"): + get_ratings( + monitoring_location_id="USGS-01104475", + time=["2026-04-29", "tomorrow"], + download_and_parse=False, + ) + assert httpx_mock.get_requests() == [] + + def test_get_ratings_multi_type_filters_via_property(httpx_mock, tmp_path): """File_type list: server filter omits it; local filter reads the property.""" httpx_mock.add_response( diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index c612e966..541f7a24 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -800,6 +800,18 @@ def test_get_daily_keeps_a_dotdot_open_bound_in_time(httpx_mock): assert sent["time"] == ["2025-01-01/.."] +@pytest.mark.parametrize( + "time", ["2025-13-45", ["2025-01-01", "yesterday"]], ids=["single", "range"] +) +def test_get_daily_rejects_an_unreadable_time_before_any_request(httpx_mock, time): + """A bound that matches no date format used to drop the whole filter, so + the service received an empty ``time=`` and answered HTTP 400 without + naming the argument. It now fails locally, naming ``time``.""" + with pytest.raises(ValueError, match=r"^time could not be read as a date"): + get_daily(monitoring_location_id="USGS-05427718", time=time) + assert httpx_mock.get_requests() == [] + + def test_get_daily_value_is_float_when_every_value_is_whole(httpx_mock): """Issue #428: whole-number values used to infer ``int64``, so ``value`` changed dtype between calls. It is ``float64`` regardless of the data.""" diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index c2e1bf22..281744c0 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -930,7 +930,6 @@ def test_type_cols_warning_is_singular_for_one_value(): ("2018-02-12T00:00:00Z/..", False, "2018-02-12T00:00:00Z/.."), ("P7D", False, "P7D"), ("PT36H", False, "PT36H"), - ("Apr", False, None), ("2024-01-01", True, "2024-01-01"), (["2024-01-01", "2024-02-01"], True, "2024-01-01/2024-02-01"), ("2024-01-01 00:00:00", True, "2024-01-01"), @@ -952,7 +951,6 @@ def test_type_cols_warning_is_singular_for_one_value(): "passthrough_interval", "passthrough_duration", "time_only_duration", - "word_with_p_not_duration", "date_only", "date_only_pair", "space_separated", @@ -966,8 +964,8 @@ def test_type_cols_warning_is_singular_for_one_value(): def test_format_api_dates(value, date, expected): """``_format_api_dates`` normalizes ISO 8601 datetimes to UTC (dropping fractional seconds, converting offsets), joins a pair into an interval, - passes durations / intervals through unchanged, renders a None endpoint as - ``..``, and returns None for a non-date word (e.g. ``"Apr"``).""" + passes durations / intervals through unchanged, and renders a None + endpoint as ``..``.""" assert _format_api_dates(value, date=date) == expected @@ -984,6 +982,30 @@ def test_format_api_dates_treats_an_all_blank_sequence_as_no_filter(): assert _format_api_dates(["..", ".."]) is None +@pytest.mark.parametrize( + "value", + ["2024-13-45", "Jan 1 2024", "Apr", ["2024-01-01", "garbage"], ["garbage", None]], + ids=[ + "single", + "unsupported_format", + # Starts with "p" but is not a duration, so it is not passed through. + "word_with_p_not_duration", + "range_end", + "range_start", + ], +) +def test_format_api_dates_rejects_an_unreadable_bound(value): + """An element that is not blank and matches no format used to return None, + which the callers send as no filter at all. The message names the caller's + argument and the bad value, and shows the forms that are accepted.""" + with pytest.raises(ValueError) as excinfo: + _format_api_dates(value, name="last_modified") + message = str(excinfo.value) + assert message.startswith("last_modified could not be read as a date") + assert "'2024-01-01'" in message + assert "None for an open end" in message + + def test_format_api_dates_rejects_more_than_two_values(): """A date filter is an instant, a duration, or a closed interval. Three values is a caller who meant something else, and the message says which