Add additional temporal aliases

Author: GomezCF

stac_fastapi/core/stac_fastapi/core/core.py

@@ -510,18 +510,20 @@ async def post_all_collections(
         return await self.all_collections(
             limit=search_request.limit if hasattr(search_request, "limit") else None,
             bbox=search_request.bbox if hasattr(search_request, "bbox") else None,
-            datetime=search_request.datetime
-            if hasattr(search_request, "datetime")
-            else None,
+            datetime=(
+                search_request.datetime if hasattr(search_request, "datetime") else None
+            ),
             token=search_request.token if hasattr(search_request, "token") else None,
             fields=fields,
             sortby=sortby,
-            filter_expr=search_request.filter
-            if hasattr(search_request, "filter")
-            else None,
-            filter_lang=search_request.filter_lang
-            if hasattr(search_request, "filter_lang")
-            else None,
+            filter_expr=(
+                search_request.filter if hasattr(search_request, "filter") else None
+            ),
+            filter_lang=(
+                search_request.filter_lang
+                if hasattr(search_request, "filter_lang")
+                else None
+            ),
             query=search_request.query if hasattr(search_request, "query") else None,
             q=search_request.q if hasattr(search_request, "q") else None,
             request=request,

stac_fastapi/core/stac_fastapi/core/version.py

@@ -1,2 +1,3 @@
 """library version."""
+
 __version__ = "6.7.4"

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py

@@ -1,2 +1,3 @@
 """library version."""
+
 __version__ = "6.7.4"

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -494,7 +494,6 @@ def apply_datetime_filter(
         if not datetime_search:
             return search, result_metadata
 
-
         if USE_DATETIME:
             if "eq" in datetime_search:
                 # For exact matches, include:

stac_fastapi/opensearch/stac_fastapi/opensearch/version.py

@@ -1,2 +1,3 @@
 """library version."""
+
 __version__ = "6.7.4"

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py

@@ -138,7 +138,7 @@ def return_date(
         return result
 
 
-def extract_date(date_str: str | None) -> date | None:
+def extract_date(date_str: str) -> date:
     """Extract date from ISO format string.
 
     Args:
@@ -147,9 +147,6 @@ def extract_date(date_str: str | None) -> date | None:
     Returns:
         A date object extracted from the input string or None.
     """
-    if not date_str:
-        return None
-
     date_str = date_str.replace("Z", "+00:00")
     return datetime_type.fromisoformat(date_str).date()
 

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py

@@ -4,11 +4,9 @@
 """
 
 import re
-from datetime import datetime
+from datetime import date, datetime
 from functools import lru_cache
-from typing import Any, List, Optional, Tuple, Dict
-
-from dateutil.parser import parse  # type: ignore[import]
+from typing import Any, Dict, List, Optional, Tuple
 
 from stac_fastapi.sfeos_helpers.mappings import (
     _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
@@ -72,37 +70,59 @@ def indices(collection_ids: Optional[List[str]]) -> str:
 
 def filter_indexes_by_datetime(
     collection_indexes: List[Tuple[Dict[str, str], ...]],
-    datetime_search: Dict[str, Dict[str, str | None]]
+    datetime_search: Dict[str, Dict[str, str | None]],
 ) -> List[str]:
+    """
+    Filter Elasticsearch index aliases based on datetime search criteria.
+
+    Filters a list of collection indexes by matching their datetime, start_datetime, and end_datetime
+    aliases against the provided search criteria. Each criterion can have optional 'gte' (greater than
+    or equal) and 'lte' (less than or equal) bounds.
+
+    Args:
+        collection_indexes (List[Tuple[Dict[str, str], ...]]): A list of tuples containing dictionaries
+            with 'datetime', 'start_datetime', and 'end_datetime' aliases.
+        datetime_search (Dict[str, Dict[str, str | None]]): A dictionary with keys 'datetime',
+            'start_datetime', and 'end_datetime', each containing 'gte' and 'lte' criteria as ISO format
+            datetime strings or None.
+
+    Returns:
+        List[str]: A list of start_datetime aliases that match all provided search criteria.
+    """
+
     def extract_date_from_alias(alias: str) -> tuple[datetime, datetime] | None:
-        date_pattern = re.compile(r'\d{4}-\d{2}-\d{2}')
+        date_pattern = re.compile(r"\d{4}-\d{2}-\d{2}")
         try:
             dates = date_pattern.findall(alias)
 
             if not dates:
                 return None
 
             if len(dates) >= 2:
-                return datetime.strptime(dates[-2], '%Y-%m-%d'), datetime.strptime(dates[-1], '%Y-%m-%d')
+                return datetime.strptime(dates[-2], "%Y-%m-%d"), datetime.strptime(
+                    dates[-1], "%Y-%m-%d"
+                )
             else:
-                date = datetime.strptime(dates[-1], '%Y-%m-%d')
+                date = datetime.strptime(dates[-1], "%Y-%m-%d")
                 return date, date
         except (ValueError, IndexError):
             return None
 
-    def parse_search_date(date_str: str | None) -> Optional[datetime.date]:
+    def parse_search_date(date_str: str | None) -> Optional[date]:
         if not date_str:
             return None
-        return datetime.fromisoformat(date_str.replace('Z', '+00:00')).date()
+        date_str = date_str.rstrip("Z")
+        return datetime.fromisoformat(date_str).date()
 
-    def check_criteria(value_begin: datetime.date, value_end: datetime.date, criteria: Dict) -> bool:
-        gte = parse_search_date(criteria.get('gte'))
-        lte = parse_search_date(criteria.get('lte'))
+    def check_criteria(value_begin: date, value_end: date, criteria: Dict) -> bool:
+        gte = parse_search_date(criteria.get("gte"))
+        lte = parse_search_date(criteria.get("lte"))
 
-        if gte and value_begin.date() < gte:
+        if gte and value_end < gte:
             return False
-        if lte and value_end.date() > lte:
+        if lte and value_begin > lte:
             return False
+
         return True
 
     filtered_indexes = []
@@ -112,9 +132,9 @@ def check_criteria(value_begin: datetime.date, value_end: datetime.date, criteri
             continue
 
         index_dict = index_tuple[0]
-        start_datetime_alias = index_dict.get('start_datetime')
-        end_datetime_alias = index_dict.get('end_datetime')
-        datetime_alias = index_dict.get('datetime')
+        start_datetime_alias = index_dict.get("start_datetime")
+        end_datetime_alias = index_dict.get("end_datetime")
+        datetime_alias = index_dict.get("datetime")
 
         if not start_datetime_alias:
             continue
@@ -123,15 +143,20 @@ def check_criteria(value_begin: datetime.date, value_end: datetime.date, criteri
         end_date = extract_date_from_alias(end_datetime_alias)
         datetime_date = extract_date_from_alias(datetime_alias)
 
-        if not start_range or not end_date or not datetime_date:
-            continue
-
-        if not check_criteria(start_range[0], start_range[1], datetime_search.get('start_datetime', {})):
-            continue
-        if not check_criteria(end_date[0], end_date[1], datetime_search.get('end_datetime', {})):
-            continue
-        if not check_criteria(datetime_date[0], datetime_date[1], datetime_search.get('datetime', {})):
+        if not check_criteria(
+            start_range[0], start_range[1], datetime_search.get("start_datetime", {})
+        ):
             continue
+        if end_date:
+            if not check_criteria(
+                end_date[0], end_date[1], datetime_search.get("end_datetime", {})
+            ):
+                continue
+        if datetime_date:
+            if not check_criteria(
+                datetime_date[0], datetime_date[1], datetime_search.get("datetime", {})
+            ):
+                continue
 
         filtered_indexes.append(start_datetime_alias)
 

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py

@@ -130,7 +130,7 @@ def apply_collections_datetime_filter_shared(
 
 
 def apply_collections_bbox_filter_shared(
-    bbox: Union[str, List[float], None]
+    bbox: Union[str, List[float], None],
 ) -> Optional[Dict[str, Dict]]:
     """Create a geo_shape filter for collections bbox search.
 

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py

@@ -1,7 +1,7 @@
 """Search engine adapters for different implementations."""
 
 import uuid
-from typing import Any, Dict, Literal, List
+from typing import Any, Dict, List, Literal
 
 from stac_fastapi.sfeos_helpers.database import (
     index_alias_by_collection_id,
@@ -39,27 +39,36 @@ async def create_simple_index(self, client: Any, collection_id: str) -> str:
         return index_name
 
     async def create_datetime_index(
-        self, client: Any, collection_id: str, start_datetime: str, datetime: str | None, end_datetime: str
+        self,
+        client: Any,
+        collection_id: str,
+        start_datetime: str,
+        datetime: str,
+        end_datetime: str,
     ) -> str:
         """Create a datetime-based index for the given collection.
 
         Args:
             client: Search engine client instance.
             collection_id (str): Collection identifier.
             start_datetime (str): Start datetime for the index alias.
-            datetime (str | None): Datetime for the datetime alias (can be None).
+            datetime (str): Datetime for the datetime alias (can be None).
             end_datetime (str): End datetime for the index alias.
 
         Returns:
             str: Created start_datetime alias name.
         """
         index_name = self.create_index_name(collection_id)
-        alias_start_date = self.create_alias_name(collection_id, "start_datetime", start_datetime)
+        alias_start_date = self.create_alias_name(
+            collection_id, "start_datetime", start_datetime
+        )
         alias_date = self.create_alias_name(collection_id, "datetime", datetime)
-        alias_end_date = self.create_alias_name(collection_id, "end_datetime", end_datetime)
+        alias_end_date = self.create_alias_name(
+            collection_id, "end_datetime", end_datetime
+        )
         collection_alias = index_alias_by_collection_id(collection_id)
 
-        aliases = {
+        aliases: Dict[str, Any] = {
             collection_alias: {},
             alias_start_date: {},
             alias_date: {},
@@ -96,7 +105,12 @@ async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str:
         return new_alias
 
     @staticmethod
-    async def change_alias_name(client: Any, old_start_datetime_alias: str, aliases_to_change: List[str], aliases_to_create: List[str]) -> None:
+    async def change_alias_name(
+        client: Any,
+        old_start_datetime_alias: str,
+        aliases_to_change: List[str],
+        aliases_to_create: List[str],
+    ) -> None:
         """Change alias names by removing old aliases and adding new ones.
 
         Args:
@@ -135,7 +149,9 @@ def create_index_name(collection_id: str) -> str:
 
     @staticmethod
     def create_alias_name(
-        collection_id: str, name: Literal["start_datetime", "datetime", "end_datetime"], start_date: str
+        collection_id: str,
+        name: Literal["start_datetime", "datetime", "end_datetime"],
+        start_date: str,
     ) -> str:
         """Create alias name from collection ID and date.
 
@@ -180,7 +196,11 @@ async def find_latest_item_in_index(client: Any, index_name: str) -> dict[str, A
         query = {
             "size": 1,
             "sort": [{"properties.start_datetime": {"order": "desc"}}],
-            "_source": ["properties.start_datetime", "properties.datetime", "properties.end_datetime"],
+            "_source": [
+                "properties.start_datetime",
+                "properties.datetime",
+                "properties.end_datetime",
+            ],
         }
 
         response = await client.search(index=index_name, body=query)

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py

@@ -1,6 +1,7 @@
 """Async index insertion strategies."""
+
 import logging
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from fastapi import HTTPException, status
 
@@ -117,7 +118,7 @@ async def _get_target_index_internal(
         collection_id: str,
         product: Dict[str, Any],
         check_size: bool = True,
-    ) -> str:
+    ) -> Optional[str]:
         """Get target index with size checking internally.
 
         Args:
@@ -132,9 +133,9 @@ async def _get_target_index_internal(
         product_datetimes = self.datetime_manager.validate_product_datetimes(product)
 
         datetime_ranges = {
-            "start_datetime": {"gte": product_datetimes.start_datetime, "lte": product_datetimes.start_datetime},
-            "end_datetime": {"gte": product_datetimes.end_datetime, "lte": product_datetimes.end_datetime},
-            "datetime": {"gte": product_datetimes.datetime, "lte": product_datetimes.datetime}
+            "start_datetime": {"gte": None, "lte": product_datetimes.start_datetime},
+            "end_datetime": {"gte": None, "lte": None},
+            "datetime": {"gte": None, "lte": None},
         }
 
         target_index = await index_selector.select_indexes(
@@ -149,7 +150,7 @@ async def _get_target_index_internal(
             await index_selector.refresh_cache()
             return target_index
 
-        all_indexes = sorted(all_indexes, key=lambda x: x[0]['start_datetime'])
+        all_indexes = sorted(all_indexes, key=lambda x: x[0]["start_datetime"])
         start_date = extract_date(product_datetimes.start_datetime)
         end_date = extract_first_date_from_index(all_indexes[0][0]["start_datetime"])
 
@@ -160,7 +161,7 @@ async def _get_target_index_internal(
             await index_selector.refresh_cache()
             return alias
 
-        if target_index != all_indexes[-1]:
+        if target_index != all_indexes[-1][0]["start_datetime"]:
             for item in all_indexes:
                 aliases_dict = item[0]
                 if target_index in aliases_dict.values():
@@ -179,8 +180,16 @@ async def _get_target_index_internal(
                         collection_id, product_datetimes, aliases_dict
                     )
                     await index_selector.refresh_cache()
+                    return target_index
 
-        return target_index
+        for item in all_indexes:
+            aliases_dict = item[0]
+            if target_index in aliases_dict.values():
+                await self.datetime_manager.handle_early_date(
+                    collection_id, product_datetimes, aliases_dict
+                )
+                return target_index
+        return None
 
     async def _ensure_indexes_exist(
         self, index_selector, collection_id: str, items: List[Dict[str, Any]]
@@ -199,9 +208,9 @@ async def _ensure_indexes_exist(
             await self.index_operations.create_datetime_index(
                 self.client,
                 collection_id,
-                extract_date(first_item["properties"]["start_datetime"]),
-                extract_date(first_item["properties"]["datetime"]),
-                extract_date(first_item["properties"]["end_datetime"]),
+                str(extract_date(first_item["properties"]["start_datetime"])),
+                str(extract_date(first_item["properties"]["datetime"])),
+                str(extract_date(first_item["properties"]["end_datetime"])),
             )
             await index_selector.refresh_cache()
 
@@ -227,9 +236,9 @@ async def _check_and_handle_oversized_index(
         )
 
         all_indexes = await index_selector.get_collection_indexes(collection_id)
-        all_indexes = sorted(all_indexes, key=lambda x: x[0]['start_datetime'])
+        all_indexes = sorted(all_indexes, key=lambda x: x[0]["start_datetime"])
 
-        latest_index = all_indexes[-1][0]
+        latest_index = all_indexes[-1][0]["start_datetime"]
 
         if first_item_index != latest_index:
             return None
@@ -248,7 +257,9 @@ async def _check_and_handle_oversized_index(
             end_datetime=latest_item["_source"]["properties"]["end_datetime"],
         )
 
-        await self.datetime_manager.handle_oversized_index(collection_id, product_datetimes, latest_index)
+        await self.datetime_manager.handle_oversized_index(
+            collection_id, product_datetimes, all_indexes[-1][0]
+        )
         await index_selector.refresh_cache()