From 3fd821d410716a6ad0246b9e36ae04ceaba7b6e0 Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Wed, 15 Oct 2025 10:51:00 +0200 Subject: [PATCH 1/3] fix: round milliseconds instead of truncating --- .../core/stac_fastapi/core/datetime_utils.py | 3 +- .../sfeos_helpers/database/datetime.py | 51 +++++++++++--- stac_fastapi/tests/api/test_api.py | 67 +++++++++++++++++++ 3 files changed, 112 insertions(+), 9 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/datetime_utils.py b/stac_fastapi/core/stac_fastapi/core/datetime_utils.py index d5f992de8..30c4842aa 100644 --- a/stac_fastapi/core/stac_fastapi/core/datetime_utils.py +++ b/stac_fastapi/core/stac_fastapi/core/datetime_utils.py @@ -23,7 +23,8 @@ def normalize(dt): return ".." dt_obj = rfc3339_str_to_datetime(dt) dt_utc = dt_obj.astimezone(timezone.utc) - return dt_utc.isoformat(timespec="milliseconds").replace("+00:00", "Z") + rounded_dt = dt_utc.replace(microsecond=round(dt_utc.microsecond / 1000) * 1000) + return rounded_dt.isoformat(timespec="milliseconds").replace("+00:00", "Z") if not isinstance(date_str, str): return "../.." diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py index d6b68e858..b58e994b9 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py @@ -8,9 +8,10 @@ import re from datetime import date from datetime import datetime as datetime_type +from datetime import timezone from typing import Dict, Optional, Union -from stac_fastapi.types.rfc3339 import DateTimeType +from stac_fastapi.types.rfc3339 import DateTimeType, rfc3339_str_to_datetime logger = logging.getLogger(__name__) @@ -36,6 +37,16 @@ def return_date( dict: A dictionary representing the date interval for use in filtering search results, always containing 'gte' and 'lte' keys. """ + + def normalize_datetime(dt_str): + """Normalize datetime string and preserve millisecond precision.""" + if not dt_str or dt_str == "..": + return dt_str + dt_obj = rfc3339_str_to_datetime(dt_str) + dt_utc = dt_obj.astimezone(timezone.utc) + rounded_dt = dt_utc.replace(microsecond=round(dt_utc.microsecond / 1000) * 1000) + return rounded_dt.isoformat(timespec="milliseconds").replace("+00:00", "Z") + result: Dict[str, Optional[str]] = {"gte": None, "lte": None} if interval is None: @@ -44,29 +55,53 @@ def return_date( if isinstance(interval, str): if "/" in interval: parts = interval.split("/") - result["gte"] = ( + gte_value = ( parts[0] if parts[0] != ".." else datetime_type.min.isoformat() + "Z" ) - result["lte"] = ( + lte_value = ( parts[1] if len(parts) > 1 and parts[1] != ".." else datetime_type.max.isoformat() + "Z" ) + + result["gte"] = ( + normalize_datetime(gte_value) + if gte_value != datetime_type.min.isoformat() + "Z" + else gte_value + ) + result["lte"] = ( + normalize_datetime(lte_value) + if lte_value != datetime_type.max.isoformat() + "Z" + else lte_value + ) else: - converted_time = interval if interval != ".." else None + converted_time = normalize_datetime(interval) if interval != ".." else ".." result["gte"] = result["lte"] = converted_time return result if isinstance(interval, datetime_type): - datetime_iso = interval.isoformat() - result["gte"] = result["lte"] = datetime_iso + datetime_str = interval.isoformat() + normalized_datetime = normalize_datetime(datetime_str) + result["gte"] = result["lte"] = normalized_datetime elif isinstance(interval, tuple): start, end = interval # Ensure datetimes are converted to UTC and formatted with 'Z' if start: - result["gte"] = start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + dt_utc = start.astimezone(timezone.utc) + rounded_dt = dt_utc.replace( + microsecond=round(dt_utc.microsecond / 1000) * 1000 + ) + result["gte"] = rounded_dt.isoformat(timespec="milliseconds").replace( + "+00:00", "Z" + ) if end: - result["lte"] = end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + dt_utc = end.astimezone(timezone.utc) + rounded_dt = dt_utc.replace( + microsecond=round(dt_utc.microsecond / 1000) * 1000 + ) + result["lte"] = rounded_dt.isoformat(timespec="milliseconds").replace( + "+00:00", "Z" + ) return result diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 6fdc2fb60..2c24e5320 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1625,3 +1625,70 @@ async def test_use_datetime_false(app_client, load_test_data, txn_client, monkey assert "test-item-datetime-only" not in found_ids assert "test-item-start-end-only" in found_ids + + +@pytest.mark.asyncio +async def test_format_datetime_range_microsecond_rounding( + app_client, txn_client, load_test_data +): + """Test that microseconds are rounded to milliseconds""" + + test_collection = load_test_data("test_collection.json") + test_collection_id = "test-collection-microseconds" + test_collection["id"] = test_collection_id + await create_collection(txn_client, test_collection) + + item = load_test_data("test_item.json") + item["id"] = "test-item-1" + item["collection"] = test_collection_id + item["properties"]["datetime"] = "2020-01-01T12:00:00.123Z" + await create_item(txn_client, item) + + test_cases = [ + ("2020-01-01T12:00:00.123678Z", False), + ("2020-01-01T12:00:00.123499Z", True), + ("2020-01-01T12:00:00.123500Z", False), + ] + + for datetime_input, should_match in test_cases: + # Test GET /collections/{id}/items + resp = await app_client.get( + f"/collections/{test_collection_id}/items", + params={"datetime": datetime_input}, + ) + assert resp.status_code == 200 + resp_json = resp.json() + + if should_match: + assert len(resp_json["features"]) == 1 + assert resp_json["features"][0]["id"] == "test-item-1" + else: + assert len(resp_json["features"]) == 0 + + # Test GET /search + resp = await app_client.get( + "/search", + params={"collections": test_collection_id, "datetime": datetime_input}, + ) + assert resp.status_code == 200 + resp_json = resp.json() + + if should_match: + assert len(resp_json["features"]) == 1 + assert resp_json["features"][0]["id"] == "test-item-1" + else: + assert len(resp_json["features"]) == 0 + + # Test POST /search + resp = await app_client.post( + "/search", + json={"collections": [test_collection_id], "datetime": datetime_input}, + ) + assert resp.status_code == 200 + resp_json = resp.json() + + if should_match: + assert len(resp_json["features"]) == 1 + assert resp_json["features"][0]["id"] == "test-item-1" + else: + assert len(resp_json["features"]) == 0 From 974e910a79f547b1309e1489264a8a40f66d5081 Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Mon, 3 Nov 2025 18:16:04 +0100 Subject: [PATCH 2/3] docs: update changelog about datetime rounding --- CHANGELOG.md | 2 ++ stac_fastapi/tests/api/test_api.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6323533ec..d06c4e8c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed +- Ensure datetime filter rounds microseconds using standard rounding instead of truncation. [#492](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/492) + ## [v6.5.1] - 2025-09-30 ### Fixed diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 2c24e5320..0f89f6456 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1631,7 +1631,7 @@ async def test_use_datetime_false(app_client, load_test_data, txn_client, monkey async def test_format_datetime_range_microsecond_rounding( app_client, txn_client, load_test_data ): - """Test that microseconds are rounded to milliseconds""" + """Test that microseconds are rounded in format_datetime_range""" test_collection = load_test_data("test_collection.json") test_collection_id = "test-collection-microseconds" From 438c2af32934b9c7e0d2f2253360bf02b31d1ab4 Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Fri, 7 Nov 2025 13:46:25 +0100 Subject: [PATCH 3/3] fix: use half round up round instread of round half to even/banker's rounding --- stac_fastapi/core/stac_fastapi/core/datetime_utils.py | 5 ++++- .../stac_fastapi/sfeos_helpers/database/datetime.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/datetime_utils.py b/stac_fastapi/core/stac_fastapi/core/datetime_utils.py index 30c4842aa..711b697ca 100644 --- a/stac_fastapi/core/stac_fastapi/core/datetime_utils.py +++ b/stac_fastapi/core/stac_fastapi/core/datetime_utils.py @@ -1,5 +1,6 @@ """Utility functions to handle datetime parsing.""" +import math from datetime import datetime, timezone from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime @@ -23,7 +24,9 @@ def normalize(dt): return ".." dt_obj = rfc3339_str_to_datetime(dt) dt_utc = dt_obj.astimezone(timezone.utc) - rounded_dt = dt_utc.replace(microsecond=round(dt_utc.microsecond / 1000) * 1000) + rounded_dt = dt_utc.replace( + microsecond=math.floor(dt_utc.microsecond / 1000 + 0.5) * 1000 + ) return rounded_dt.isoformat(timespec="milliseconds").replace("+00:00", "Z") if not isinstance(date_str, str): diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py index b58e994b9..bf861e922 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py @@ -5,6 +5,7 @@ """ import logging +import math import re from datetime import date from datetime import datetime as datetime_type @@ -44,7 +45,9 @@ def normalize_datetime(dt_str): return dt_str dt_obj = rfc3339_str_to_datetime(dt_str) dt_utc = dt_obj.astimezone(timezone.utc) - rounded_dt = dt_utc.replace(microsecond=round(dt_utc.microsecond / 1000) * 1000) + rounded_dt = dt_utc.replace( + microsecond=math.floor(dt_utc.microsecond / 1000 + 0.5) * 1000 + ) return rounded_dt.isoformat(timespec="milliseconds").replace("+00:00", "Z") result: Dict[str, Optional[str]] = {"gte": None, "lte": None}