Merge branch 'main' into collection_update_endpoint

Author: rhysrevans3

.github/workflows/cicd.yml

@@ -16,7 +16,7 @@ jobs:
     services:
 
       elasticsearch_8_svc:
-        image: docker.elastic.co/elasticsearch/elasticsearch:8.1.3
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
         env:
           cluster.name: stac-cluster
           node.name: es01
@@ -58,7 +58,7 @@ jobs:
 
       # Setup Python (faster than using Python container)
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Lint code

CHANGELOG.md

@@ -9,18 +9,39 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
-- Collection-level Assets to the CollectionSerializer [#148](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/148)
+### Changed
+
+- Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169)
+
+### Fixed
+
+- Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166)
+- Upgrade stac-fastapi to v2.4.9 [#172](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/172)
+
+## [v1.0.0]
 
 ### Added
 
+- Collection-level Assets to the CollectionSerializer [#148](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/148)
+- Pagination for /collections - GET all collections - route [#164](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/164)
 - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147)
+- GET /search filter extension queries [#163](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/163)
+- Added support for GET /search intersection queries [#158](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/158)
 
 ### Changed
+
+- Update elasticsearch version from 8.1.3 to 8.10.4 in cicd, gh actions [#164](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/164)
+- Updated core stac-fastapi libraries to 2.4.8 from 2.4.3 [#151](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/151)
+- Use aliases on Elasticsearch indices, add number suffix in index name. [#152](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/152)
+
 ### Fixed
 
 - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132)
 - Corrected the automatic converstion of float values to int when building Filter Clauses [#135](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/135)
+- Do not index `proj:geometry` field as geo_shape [#154](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/154)
 - Remove unsupported characters from Elasticsearch index names [#153](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/153)
+- Fixed GET /search sortby requests [#25](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/25)
+
 
 ## [v0.3.0]
 
@@ -34,11 +55,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Fields Extension [#129](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/129)
 - Support for Python 3.11 [#131](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/131)
 
-
 ### Changed
 
 - Updated core stac-fastapi libraries to 2.4.3 from 2.3.0 [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127)
 
+
 ## [v0.2.0]
 
 ### Added
@@ -61,6 +82,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - When using bulk ingest, items will continue to be ingested if any of them fail. Previously, the call would fail
   immediately if any items failed.
 
+
 ## [v0.1.0]
 
 ### Changed
@@ -72,7 +94,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Use genexp in execute_search and get_all_collections to return results.
 - Added db_to_stac serializer to item_collection method in core.py.
 
-[Unreleased]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.3.0...main>
+
+[Unreleased]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.0.0...main>
+[v1.0.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.3.0...v1.0.0>
 [v0.3.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.2.0...v0.3.0>
 [v0.2.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0...v0.2.0>
 [v0.1.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0>

README.md

@@ -61,7 +61,17 @@ curl -X "POST" "http://localhost:8080/collections" \
 ```
 
 Note: this "Collections Transaction" behavior is not part of the STAC API, but may be soon.  
+
+
+## Collection pagination
+
+The collections route handles optional `limit` and `token` parameters. The `links` field that is
+returned from the `/collections` route contains a `next` link with the token that can be used to 
+get the next page of results.
 
+```shell
+curl -X "GET" "http://localhost:8080/collections?limit=1&token=example_token"
+```
 
 ## Testing
 

VERSION

@@ -1 +1 @@
-0.3.0
+1.0.0

docker-compose.yml

@@ -31,7 +31,7 @@ services:
 
   elasticsearch:
     container_name: es-container
-    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.1.3}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
     environment:
       ES_JAVA_OPTS: -Xms512m -Xmx1g
     volumes:

stac_fastapi/elasticsearch/setup.py

@@ -8,15 +8,16 @@
 install_requires = [
     "fastapi",
     "attrs",
-    "pydantic[dotenv]",
+    "pydantic[dotenv]<2",
     "stac_pydantic==2.0.*",
-    "stac-fastapi.types==2.4.3",
-    "stac-fastapi.api==2.4.3",
-    "stac-fastapi.extensions==2.4.3",
-    "elasticsearch[async]==7.17.9",
-    "elasticsearch-dsl==7.4.1",
+    "stac-fastapi.types==2.4.9",
+    "stac-fastapi.api==2.4.9",
+    "stac-fastapi.extensions==2.4.9",
+    "elasticsearch[async]==8.11.0",
+    "elasticsearch-dsl==8.11.0",
     "pystac[validation]",
     "uvicorn",
+    "orjson",
     "overrides",
     "starlette",
     "geojson-pydantic",

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py

@@ -1,8 +1,4 @@
 """FastAPI application."""
-from typing import List
-
-import attr
-
 from stac_fastapi.api.app import StacApi
 from stac_fastapi.api.models import create_get_request_model, create_post_request_model
 from stac_fastapi.elasticsearch.config import ElasticsearchSettings
@@ -28,52 +24,15 @@
 settings = ElasticsearchSettings()
 session = Session.create_from_settings(settings)
 
-
-@attr.s
-class FixedSortExtension(SortExtension):
-    """SortExtension class fixed with correct paths, removing extra forward-slash."""
-
-    conformance_classes: List[str] = attr.ib(
-        factory=lambda: ["https://api.stacspec.org/v1.0.0-beta.4/item-search#sort"]
-    )
-
-
-@attr.s
-class FixedFilterExtension(FilterExtension):
-    """FilterExtension class fixed with correct paths, removing extra forward-slash."""
-
-    conformance_classes: List[str] = attr.ib(
-        default=[
-            "https://api.stacspec.org/v1.0.0-rc.1/item-search#filter",
-            "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/filter",
-            "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/features-filter",
-            "http://www.opengis.net/spec/cql2/1.0/conf/cql2-text",
-            "http://www.opengis.net/spec/cql2/1.0/conf/cql2-json",
-            "http://www.opengis.net/spec/cql2/1.0/conf/basic-cql2",
-            "http://www.opengis.net/spec/cql2/1.0/conf/basic-spatial-operators",
-        ]
-    )
-    client = attr.ib(factory=EsAsyncBaseFiltersClient)
-
-
-@attr.s
-class FixedQueryExtension(QueryExtension):
-    """Fixed Query Extension string."""
-
-    conformance_classes: List[str] = attr.ib(
-        factory=lambda: ["https://api.stacspec.org/v1.0.0-beta.4/item-search#query"]
-    )
-
-
 extensions = [
     TransactionExtension(client=TransactionsClient(session=session), settings=settings),
     BulkTransactionExtension(client=BulkTransactionsClient(session=session)),
     FieldsExtension(),
-    FixedQueryExtension(),
-    FixedSortExtension(),
+    QueryExtension(),
+    SortExtension(),
     TokenPaginationExtension(),
     ContextExtension(),
-    FixedFilterExtension(),
+    FilterExtension(client=EsAsyncBaseFiltersClient()),
 ]
 
 post_request_model = create_post_request_model(extensions)

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py

@@ -1,30 +1,43 @@
 """API configuration."""
 import os
+import ssl
 from typing import Any, Dict, Set
 
 from elasticsearch import AsyncElasticsearch, Elasticsearch  # type: ignore
 from stac_fastapi.types.config import ApiSettings
 
 
 def _es_config() -> Dict[str, Any]:
+    # Determine the scheme (http or https)
+    use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true"
+    scheme = "https" if use_ssl else "http"
+
+    # Configure the hosts parameter with the correct scheme
+    hosts = [f"{scheme}://{os.getenv('ES_HOST')}:{os.getenv('ES_PORT')}"]
+
+    # Initialize the configuration dictionary
     config = {
-        "hosts": [{"host": os.getenv("ES_HOST"), "port": os.getenv("ES_PORT")}],
+        "hosts": hosts,
         "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"},
-        "use_ssl": True,
-        "verify_certs": True,
     }
 
-    if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")):
-        config["http_auth"] = (u, p)
+    # Explicitly exclude SSL settings when not using SSL
+    if not use_ssl:
+        return config
 
-    if (v := os.getenv("ES_USE_SSL")) and v == "false":
-        config["use_ssl"] = False
+    # Include SSL settings if using https
+    config["ssl_version"] = ssl.TLSVersion.TLSv1_3  # type: ignore
+    config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false"  # type: ignore
 
-    if (v := os.getenv("ES_VERIFY_CERTS")) and v == "false":
-        config["verify_certs"] = False
+    # Include CA Certificates if verifying certs
+    if config["verify_certs"]:
+        config["ca_certs"] = os.getenv(
+            "CURL_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt"
+        )
 
-    if v := os.getenv("CURL_CA_BUNDLE"):
-        config["ca_certs"] = v
+    # Handle authentication
+    if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")):
+        config["http_auth"] = (u, p)
 
     return config