sort body param, cicd

Author: jamesfisher-geo

.github/workflows/cicd.yml

@@ -46,6 +46,22 @@ jobs:
           ES_JAVA_OPTS: -Xms512m -Xmx1g
         ports:
           - 9400:9400
+
+      opensearch_2_11:
+        image: opensearchproject/opensearch:2.11.1
+        env:
+          cluster.name: stac-cluster
+          node.name: os01
+          network.host: 0.0.0.0
+          transport.host: 0.0.0.0
+          discovery.type: single-node
+          http.port: 9202
+          http.cors.enabled: true
+          plugins.security.disabled: true
+          plugins.security.ssl.http.enabled: true
+          OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
+        ports:
+          - 9202:9202
     strategy:
       matrix:
         python-version: [ "3.8", "3.9", "3.10", "3.11"]
@@ -90,4 +106,15 @@ jobs:
           ES_PORT: 9400
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
-          ES_VERIFY_CERTS: false
+          ES_VERIFY_CERTS: false
+
+      - name: Run test suite against OpenSearch 2.11.1
+        run: |
+          cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
+        env:
+          ENVIRONMENT: testing
+          ES_PORT: 9202
+          ES_HOST: 172.17.0.1
+          ES_USE_SSL: false
+          ES_VERIFY_CERTS: false
+          BACKEND: opensearch

Makefile

@@ -52,7 +52,7 @@ docker-run: image-dev
 docker-shell:
 	$(run_es) /bin/bash
 
-.PHONY: test
+.PHONY: test-elasticsearch
 test:
 	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
 	docker-compose down
@@ -62,6 +62,14 @@ test-opensearch:
 	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
 	docker-compose down
 
+.PHONY: test
+test:
+	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
 .PHONY: run-database-es
 run-database-es:
 	docker-compose run --rm elasticsearch

README.md

@@ -80,11 +80,19 @@ curl -X "GET" "http://localhost:8080/collections?limit=1&token=example_token"
 ```shell
 make test
 ```
+Test against OpenSearch only
 
 ```shell
 make test-opensearch
 ```
-   
+
+Test against Elasticsearch only
+
+```shell
+make test-elasticsearch
+```  
+
+
 ## Ingest sample data
 
 ```shell

opensearch/config/opensearch.yml

@@ -17,25 +17,3 @@ plugins.security.disabled: true
 plugins.security.ssl.http.enabled: true
 
 node.max_local_storage_nodes: 3
-
-######## Start OpenSearch Security Demo Configuration ########
-# WARNING: revise all the lines below before you go into production
-# plugins.security.ssl.transport.pemcert_filepath: esnode.pem
-# plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem
-# plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
-# plugins.security.ssl.transport.enforce_hostname_verification: false
-# plugins.security.ssl.http.pemcert_filepath: esnode.pem
-# plugins.security.ssl.http.pemkey_filepath: esnode-key.pem
-# plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
-# plugins.security.allow_unsafe_democertificates: true
-# plugins.security.allow_default_init_securityindex: true
-# plugins.security.authcz.admin_dn:
-#   - CN=kirk,OU=client,O=client,L=test, C=de
-
-# plugins.security.audit.type: internal_opensearch
-# plugins.security.enable_snapshot_restore_privilege: true
-# plugins.security.check_snapshot_restore_write_privileges: true
-# plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
-# plugins.security.system_indices.enabled: true
-# plugins.security.system_indices.indices: [".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]
-######## End OpenSearch Security Demo Configuration ########

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py

@@ -110,6 +110,7 @@ async def all_collections(self, **kwargs) -> Collections:
         next_link = None
         if len(hits) == limit:
             last_hit = hits[-1]
+            logger.info(last_hit)
             next_search_after = last_hit["sort"]
             next_token = urlsafe_b64encode(
                 ",".join(map(str, next_search_after)).encode()

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic/database_logic_opensearch.py

@@ -7,6 +7,7 @@
 
 import attr
 from opensearchpy.helpers.search import Search
+from opensearchpy.helpers.query import Q
 from opensearchpy import helpers, exceptions
 from opensearchpy.exceptions import TransportError
 
@@ -336,11 +337,12 @@ async def get_all_collections(
         if token:
             search_after = urlsafe_b64decode(token.encode()).decode().split(",")
             search_body["search_after"] = search_after
+        search_body["sort"] = {"id": {"order": "asc"}}
+        
         collections = await self.client.search(
             index=COLLECTIONS_INDEX,
             body=search_body,
-            size=limit,
-            sort={"id": {"order": "asc"}},
+            size=limit
         )
         hits = collections["hits"]["hits"]
         return hits
@@ -545,10 +547,12 @@ async def execute_search(
         """
         search_body = {}
         query = search.query.to_dict() if search.query else None
-        search_body["query"] = query if query else {}
+        if query:
+            search_body["query"] = query 
         if token:
             search_after = urlsafe_b64decode(token.encode()).decode().split(",")
             search_body["search_after"] = search_after
+        search_body["sort"]  = sort if sort else DEFAULT_SORT
 
         index_param = indices(collection_ids)
 
@@ -557,8 +561,7 @@ async def execute_search(
                 index=index_param,
                 ignore_unavailable=ignore_unavailable,
                 body=search_body,
-                sort=sort or DEFAULT_SORT,
-                size=limit,
+                size=limit
             )
         )