Merge branch 'main' into collection-extensions

Author: nkleinbaer

.github/workflows/cicd.yml

@@ -46,6 +46,22 @@ jobs:
           ES_JAVA_OPTS: -Xms512m -Xmx1g
         ports:
           - 9400:9400
+
+      opensearch_2_11:
+        image: opensearchproject/opensearch:2.11.1
+        env:
+          cluster.name: stac-cluster
+          node.name: os01
+          network.host: 0.0.0.0
+          transport.host: 0.0.0.0
+          discovery.type: single-node
+          http.port: 9202
+          http.cors.enabled: true
+          plugins.security.disabled: true
+          plugins.security.ssl.http.enabled: true
+          OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
+        ports:
+          - 9202:9202
     strategy:
       matrix:
         python-version: [ "3.8", "3.9", "3.10", "3.11"]
@@ -90,4 +106,15 @@ jobs:
           ES_PORT: 9400
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
-          ES_VERIFY_CERTS: false
+          ES_VERIFY_CERTS: false
+
+      - name: Run test suite against OpenSearch 2.11.1
+        run: |
+          cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
+        env:
+          ENVIRONMENT: testing
+          ES_PORT: 9202
+          ES_HOST: 172.17.0.1
+          ES_USE_SSL: false
+          ES_VERIFY_CERTS: false
+          BACKEND: opensearch

CHANGELOG.md

@@ -9,11 +9,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+### Changed
+
+### Fixed
+
+## [v1.1.0]
+
+### Added
+
 - Advanced comparison (LIKE, IN, BETWEEN) operators to the Filter extension [#178](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/178)
+- Collection update endpoint no longer delete all sub items [#177](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/177)
+- OpenSearch 2.11.1 support [#188](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/188)
 
 ### Changed
 
 - Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169)
+- Collection update endpoint no longer delete all sub items [#177](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/177)
 
 ### Fixed
 
@@ -99,7 +110,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Added db_to_stac serializer to item_collection method in core.py.
 
 
-[Unreleased]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.0.0...main>
+[Unreleased]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.1.0...main>
+[v1.1.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.0.0...v1.1.0>
 [v1.0.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.3.0...v1.0.0>
 [v0.3.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.2.0...v0.3.0>
 [v0.2.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0...v0.2.0>

Makefile

@@ -1,20 +1,32 @@
 #!make
 APP_HOST ?= 0.0.0.0
-APP_PORT ?= 8080
+ES_APP_PORT ?= 8080
 EXTERNAL_APP_PORT ?= ${APP_PORT}
 
-APP_PORT ?= 8080
+ES_APP_PORT ?= 8080
 ES_HOST ?= docker.for.mac.localhost
 ES_PORT ?= 9200
 
+OS_APP_PORT ?= 8082
+ES_HOST ?= docker.for.mac.localhost
+OS_PORT ?= 9202
+
 run_es = docker-compose \
 	run \
-	-p ${EXTERNAL_APP_PORT}:${APP_PORT} \
+	-p ${EXTERNAL_APP_PORT}:${ES_APP_PORT} \
 	-e PY_IGNORE_IMPORTMISMATCH=1 \
 	-e APP_HOST=${APP_HOST} \
-	-e APP_PORT=${APP_PORT} \
+	-e APP_PORT=${ES_APP_PORT} \
 	app-elasticsearch
 
+run_os = docker-compose \
+	run \
+	-p ${EXTERNAL_APP_PORT}:${OS_APP_PORT} \
+	-e PY_IGNORE_IMPORTMISMATCH=1 \
+	-e APP_HOST=${APP_HOST} \
+	-e APP_PORT=${OS_APP_PORT} \
+	app-opensearch
+
 .PHONY: image-deploy
 image-deploy:
 	docker build -f Dockerfile.deploy -t stac-fastapi-elasticsearch:latest .
@@ -40,15 +52,32 @@ docker-run: image-dev
 docker-shell:
 	$(run_es) /bin/bash
 
+.PHONY: test-elasticsearch
+test:
+	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
+.PHONY: test-opensearch
+test-opensearch:
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
 .PHONY: test
 test:
 	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
 	docker-compose down
 
-.PHONY: run-database
-run-database:
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
+.PHONY: run-database-es
+run-database-es:
 	docker-compose run --rm elasticsearch
 
+.PHONY: run-database-os
+run-database-os:
+	docker-compose run --rm opensearch
+
 .PHONY: pybase-install
 pybase-install:
 	pip install wheel && \

README.md

@@ -1,6 +1,6 @@
 # STAC FastAPI Elasticsearch (sfes)
 
-## Elasticsearch backend for stac-fastapi
+## Elasticsearch backend for stac-fastapi with Opensearch support
 
 #### Join our [Gitter](https://gitter.im/stac-fastapi-elasticsearch/community) page
 
@@ -42,13 +42,15 @@ docker-compose build
 docker-compose up
 ```
 
-By default, docker-compose uses Elasticsearch 8.x. However, most recent 7.x versions should also work. 
+By default, docker-compose uses Elasticsearch 8.x and OpenSearch 2.11.1. 
 If you wish to use a different version, put the following in a 
 file named `.env` in the same directory you run docker-compose from:
 
 ```shell
 ELASTICSEARCH_VERSION=7.17.1
+OPENSEARCH_VERSION=2.11.0
 ```
+The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information.
 
 To create a new Collection:
 
@@ -78,7 +80,18 @@ curl -X "GET" "http://localhost:8080/collections?limit=1&token=example_token"
 ```shell
 make test
 ```
-   
+Test against OpenSearch only
+
+```shell
+make test-opensearch
+```
+
+Test against Elasticsearch only
+
+```shell
+make test-elasticsearch
+```  
+
 ## Ingest sample data
 
 ```shell

VERSION

@@ -1,12 +1,24 @@
 """Database ingestion script."""
 import json
 import os
+import sys
 
 import click
 import requests
 
+if len(sys.argv) != 2:
+    print("Usage: python data_loader.py <opensearch|elasticsearch>")
+    sys.exit(1)
+
 DATA_DIR = os.path.join(os.path.dirname(__file__), "setup_data/")
-STAC_API_BASE_URL = "http://localhost:8080"
+
+backend = sys.argv[1].lower()
+if backend == "opensearch":
+    STAC_API_BASE_URL = "http://localhost:8082"
+elif backend == "elasticsearch":
+    STAC_API_BASE_URL = "http://localhost:8080"
+else:
+    print("Invalid backend tag. Enter either 'opensearch' or 'elasticsearch'.")
 
 
 def load_data(filename):

data_loader/data_loader.py

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   app-elasticsearch:
     container_name: stac-fastapi-es
-    image: stac-utils/stac-fastapi
+    image: stac-utils/stac-fastapi-es
     restart: always
     build:
       context: .
@@ -14,10 +14,11 @@ services:
       - RELOAD=true
       - ENVIRONMENT=local
       - WEB_CONCURRENCY=10
-      - ES_HOST=172.17.0.1
+      - ES_HOST=elasticsearch
       - ES_PORT=9200
       - ES_USE_SSL=false
       - ES_VERIFY_CERTS=false
+      - BACKEND=elasticsearch
     ports:
       - "8080:8080"
     volumes:
@@ -29,13 +30,57 @@ services:
     command:
       bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app"
 
+  app-opensearch:
+    container_name: stac-fastapi-os
+    image: stac-utils/stac-fastapi-os
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    environment:
+      - APP_HOST=0.0.0.0
+      - APP_PORT=8082
+      - RELOAD=true
+      - ENVIRONMENT=local
+      - WEB_CONCURRENCY=10
+      - ES_HOST=opensearch
+      - ES_PORT=9202
+      - ES_USE_SSL=false
+      - ES_VERIFY_CERTS=false
+      - BACKEND=opensearch
+    ports:
+      - "8082:8082"
+    volumes:
+      - ./stac_fastapi:/app/stac_fastapi
+      - ./scripts:/app/scripts
+      - ./osdata:/usr/share/opensearch/data
+    depends_on:
+      - opensearch
+    command:
+      bash -c "./scripts/wait-for-it-es.sh os-container:9202 && python -m stac_fastapi.elasticsearch.app"
+
   elasticsearch:
     container_name: es-container
     image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
+    hostname: elasticsearch
     environment:
       ES_JAVA_OPTS: -Xms512m -Xmx1g
     volumes:
       - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots
     ports:
       - "9200:9200"
+
+  opensearch:
+    container_name: os-container
+    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-2.11.1}
+    hostname: opensearch
+    environment:
+      - discovery.type=single-node
+      - plugins.security.disabled=true
+      - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
+    volumes:
+      - ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
+      - ./opensearch/snapshots:/usr/share/opensearch/snapshots
+    ports:
+      - "9202:9202"

docker-compose.yml

@@ -0,0 +1,19 @@
+## Cluster Settings
+cluster.name: stac-cluster
+node.name: os01
+network.host: 0.0.0.0
+transport.host: 0.0.0.0
+discovery.type: single-node
+http.port: 9202
+http.cors.enabled: true
+http.cors.allow-headers: X-Requested-With,Content-Type,Content-Length,Accept,Authorization
+
+path:
+  repo:
+    - /usr/share/opensearch/snapshots
+
+# Security
+plugins.security.disabled: true
+plugins.security.ssl.http.enabled: true
+
+node.max_local_storage_nodes: 3

opensearch/config/opensearch.yml

@@ -1,2 +1,2 @@
 [metadata]
-version = attr: stac_fastapi.elasticsearch.version.__version__
+version = 1.1.0

stac_fastapi/elasticsearch/setup.cfg

@@ -15,6 +15,8 @@
     "stac-fastapi.extensions==2.4.9",
     "elasticsearch[async]==8.11.0",
     "elasticsearch-dsl==8.11.0",
+    "opensearch-py==2.4.2",
+    "opensearch-py[async]==2.4.2",
     "pystac[validation]",
     "uvicorn",
     "orjson",
@@ -40,7 +42,7 @@
 
 setup(
     name="stac-fastapi.elasticsearch",
-    description="An implementation of STAC API based on the FastAPI framework with Elasticsearch.",
+    description="An implementation of STAC API based on the FastAPI framework with both Elasticsearch and Opensearch.",
     long_description=desc,
     long_description_content_type="text/markdown",
     python_requires=">=3.8",