Skip to content

Commit afbeb20

Browse files
jamesfisher-geojamesfisher-geo
committed
Opensearch database_logic, conditional imports. OS tests failing
1 parent ae9b38f commit afbeb20

File tree

12 files changed

+193
-122
lines changed

12 files changed

+193
-122
lines changed

‎Makefile‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,32 @@
11
#!make
22
APP_HOST ?= 0.0.0.0
3-
APP_PORT ?= 8080
3+
ES_APP_PORT ?= 8080
44
EXTERNAL_APP_PORT ?= ${APP_PORT}
55

6-
APP_PORT ?= 8080
6+
ES_APP_PORT ?= 8080
77
ES_HOST ?= docker.for.mac.localhost
88
ES_PORT ?= 9200
99

10+
OS_APP_PORT ?= 8082
11+
ES_HOST ?= docker.for.mac.localhost
12+
OS_PORT ?= 9202
13+
1014
run_es = docker-compose \
1115
run \
12-
-p ${EXTERNAL_APP_PORT}:${APP_PORT} \
16+
-p ${EXTERNAL_APP_PORT}:${ES_APP_PORT} \
1317
-e PY_IGNORE_IMPORTMISMATCH=1 \
1418
-e APP_HOST=${APP_HOST} \
15-
-e APP_PORT=${APP_PORT} \
19+
-e APP_PORT=${ES_APP_PORT} \
1620
app-elasticsearch
1721

22+
run_os = docker-compose \
23+
run \
24+
-p ${EXTERNAL_APP_PORT}:${OS_APP_PORT} \
25+
-e PY_IGNORE_IMPORTMISMATCH=1 \
26+
-e APP_HOST=${APP_HOST} \
27+
-e APP_PORT=${OS_APP_PORT} \
28+
app-opensearch
29+
1830
.PHONY: image-deploy
1931
image-deploy:
2032
docker build -f Dockerfile.deploy -t stac-fastapi-elasticsearch:latest .
@@ -45,6 +57,11 @@ test:
4557
-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
4658
docker-compose down
4759

60+
.PHONY: test-opensearch
61+
test-opensearch:
62+
-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/clients && pytest'
63+
docker-compose down
64+
4865
.PHONY: run-database
4966
run-database:
5067
docker-compose run --rm elasticsearch

‎data_loader/data_loader.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,24 @@
11
"""Database ingestion script."""
22
import json
33
import os
4+
import sys
45

56
import click
67
import requests
78

9+
if len(sys.argv) != 2:
10+
print("Usage: python data_loader.py <opensearch|elasticsearch>")
11+
sys.exit(1)
12+
813
DATA_DIR = os.path.join(os.path.dirname(__file__), "setup_data/")
9-
STAC_API_BASE_URL = "http://localhost:8080"
14+
15+
backend = sys.argv[1].lower()
16+
if backend == "opensearch":
17+
STAC_API_BASE_URL = "http://localhost:8082"
18+
elif backend == "elasticsearch":
19+
STAC_API_BASE_URL = "http://localhost:8080"
20+
else:
21+
print("Invalid backend tag. Enter either 'opensearch' or 'elasticsearch'.")
1022

1123

1224
def load_data(filename):

‎docker-compose.yml‎

Lines changed: 50 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,34 @@
11
version: '3.9'
22

33
services:
4-
# app-elasticsearch:
5-
# container_name: stac-fastapi-es
6-
# image: stac-utils/stac-fastapi-es
7-
# restart: always
8-
# build:
9-
# context: .
10-
# dockerfile: Dockerfile.dev
11-
# environment:
12-
# - APP_HOST=0.0.0.0
13-
# - APP_PORT=8080
14-
# - RELOAD=true
15-
# - ENVIRONMENT=local
16-
# - WEB_CONCURRENCY=10
17-
# - ES_HOST=172.17.0.1
18-
# - ES_PORT=9200
19-
# - ES_USE_SSL=false
20-
# - ES_VERIFY_CERTS=false
21-
# ports:
22-
# - "8080:8080"
23-
# volumes:
24-
# - ./stac_fastapi:/app/stac_fastapi
25-
# - ./scripts:/app/scripts
26-
# - ./esdata:/usr/share/elasticsearch/data
27-
# depends_on:
28-
# - elasticsearch
29-
# command:
30-
# bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app"
4+
app-elasticsearch:
5+
container_name: stac-fastapi-es
6+
image: stac-utils/stac-fastapi-es
7+
restart: always
8+
build:
9+
context: .
10+
dockerfile: Dockerfile.dev
11+
environment:
12+
- APP_HOST=0.0.0.0
13+
- APP_PORT=8080
14+
- RELOAD=true
15+
- ENVIRONMENT=local
16+
- WEB_CONCURRENCY=10
17+
- ES_HOST=172.17.0.1
18+
- ES_PORT=9200
19+
- ES_USE_SSL=false
20+
- ES_VERIFY_CERTS=false
21+
- BACKEND=elasticsearch
22+
ports:
23+
- "8080:8080"
24+
volumes:
25+
- ./stac_fastapi:/app/stac_fastapi
26+
- ./scripts:/app/scripts
27+
- ./esdata:/usr/share/elasticsearch/data
28+
depends_on:
29+
- elasticsearch
30+
command:
31+
bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app"
3132

3233
app-opensearch:
3334
container_name: stac-fastapi-os
@@ -38,44 +39,46 @@ services:
3839
dockerfile: Dockerfile.dev
3940
environment:
4041
- APP_HOST=0.0.0.0
41-
- APP_PORT=8080
42+
- APP_PORT=8082
4243
- RELOAD=true
4344
- ENVIRONMENT=local
4445
- WEB_CONCURRENCY=10
45-
- OS_HOST=172.17.0.1
46-
- OS_PORT=9200
47-
- OS_USE_SSL=false
48-
- OS_VERIFY_CERTS=false
46+
- ES_HOST=172.17.0.1
47+
- ES_PORT=9202
48+
- ES_USE_SSL=false
49+
- ES_VERIFY_CERTS=false
50+
- BACKEND=opensearch
4951
ports:
50-
- "8082:8080"
52+
- "8082:8082"
5153
volumes:
5254
- ./stac_fastapi:/app/stac_fastapi
5355
- ./scripts:/app/scripts
5456
- ./osdata:/usr/share/opensearch/data
5557
depends_on:
5658
- opensearch
5759
command:
58-
bash -c "./scripts/wait-for-it-es.sh os-container:9200 && python -m stac_fastapi.elasticsearch.app"
60+
bash -c "./scripts/wait-for-it-es.sh os-container:9202 && python -m stac_fastapi.elasticsearch.app"
5961

60-
# elasticsearch:
61-
# container_name: es-container
62-
# image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
63-
# environment:
64-
# ES_JAVA_OPTS: -Xms512m -Xmx1g
65-
# volumes:
66-
# - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
67-
# - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots
68-
# ports:
69-
# - "9200:9200"
62+
elasticsearch:
63+
container_name: es-container
64+
image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
65+
environment:
66+
ES_JAVA_OPTS: -Xms512m -Xmx1g
67+
volumes:
68+
- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
69+
- ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots
70+
ports:
71+
- "9200:9200"
7072

7173
opensearch:
7274
container_name: os-container
7375
image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-2.11.1}
7476
environment:
75-
- "discovery.type=single-node"
76-
- "plugins.security.disabled=true"
77+
- discovery.type=single-node
78+
- plugins.security.disabled=true
79+
- OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
7780
volumes:
7881
- ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
7982
- ./opensearch/snapshots:/usr/share/opensearch/snapshots
8083
ports:
81-
- "9202:9200"
84+
- "9202:9202"

‎opensearch/config/opensearch.yml‎

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,38 @@ node.name: os01
44
network.host: 0.0.0.0
55
transport.host: 0.0.0.0
66
discovery.type: single-node
7-
http.port: 9200
7+
http.port: 9202
8+
http.cors.enabled: true
9+
http.cors.allow-headers: X-Requested-With,Content-Type,Content-Length,Accept,Authorization
810

911
path:
1012
repo:
1113
- /usr/share/opensearch/snapshots
1214

13-
######## Start OpenSearch Security Demo Configuration ########
14-
# WARNING: revise all the lines below before you go into production
15-
plugins.security.ssl.transport.pemcert_filepath: esnode.pem
16-
plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem
17-
plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
18-
plugins.security.ssl.transport.enforce_hostname_verification: false
15+
# Security
16+
plugins.security.disabled: true
1917
plugins.security.ssl.http.enabled: true
20-
plugins.security.ssl.http.pemcert_filepath: esnode.pem
21-
plugins.security.ssl.http.pemkey_filepath: esnode-key.pem
22-
plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
23-
plugins.security.allow_unsafe_democertificates: true
24-
plugins.security.allow_default_init_securityindex: true
25-
plugins.security.authcz.admin_dn:
26-
- CN=kirk,OU=client,O=client,L=test, C=de
2718

28-
plugins.security.audit.type: internal_opensearch
29-
plugins.security.enable_snapshot_restore_privilege: true
30-
plugins.security.check_snapshot_restore_write_privileges: true
31-
plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
32-
plugins.security.system_indices.enabled: true
33-
plugins.security.system_indices.indices: [".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]
3419
node.max_local_storage_nodes: 3
20+
21+
######## Start OpenSearch Security Demo Configuration ########
22+
# WARNING: revise all the lines below before you go into production
23+
# plugins.security.ssl.transport.pemcert_filepath: esnode.pem
24+
# plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem
25+
# plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
26+
# plugins.security.ssl.transport.enforce_hostname_verification: false
27+
# plugins.security.ssl.http.pemcert_filepath: esnode.pem
28+
# plugins.security.ssl.http.pemkey_filepath: esnode-key.pem
29+
# plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
30+
# plugins.security.allow_unsafe_democertificates: true
31+
# plugins.security.allow_default_init_securityindex: true
32+
# plugins.security.authcz.admin_dn:
33+
# - CN=kirk,OU=client,O=client,L=test, C=de
34+
35+
# plugins.security.audit.type: internal_opensearch
36+
# plugins.security.enable_snapshot_restore_privilege: true
37+
# plugins.security.check_snapshot_restore_write_privileges: true
38+
# plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
39+
# plugins.security.system_indices.enabled: true
40+
# plugins.security.system_indices.indices: [".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]
3541
######## End OpenSearch Security Demo Configuration ########

‎stac_fastapi/elasticsearch/setup.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
"stac-fastapi.extensions==2.4.9",
1616
"elasticsearch[async]==8.11.0",
1717
"elasticsearch-dsl==8.11.0",
18-
"opensearch-py==2.1.1",
19-
"opensearch-py[async]==2.1.1",
18+
"opensearch-py==2.4.2",
19+
"opensearch-py[async]==2.4.2",
2020
"pystac[validation]",
2121
"uvicorn",
2222
"orjson",

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
11
"""FastAPI application."""
22
from stac_fastapi.api.app import StacApi
33
from stac_fastapi.api.models import create_get_request_model, create_post_request_model
4-
from stac_fastapi.elasticsearch.config.config_elasticsearch import ElasticsearchSettings
54
from stac_fastapi.elasticsearch.core import (
65
BulkTransactionsClient,
76
CoreClient,
87
EsAsyncBaseFiltersClient,
98
TransactionsClient,
109
)
11-
from stac_fastapi.elasticsearch.database_elasticsearch.database_logic.database_logic_elasticsearch import create_collection_index
10+
import os
11+
12+
backend = os.getenv("BACKEND", "elasticsearch").lower()
13+
if backend == "opensearch":
14+
from stac_fastapi.elasticsearch.config.config_opensearch import SearchSettings
15+
from stac_fastapi.elasticsearch.database_logic.database_logic_opensearch import create_collection_index
16+
else:
17+
from stac_fastapi.elasticsearch.config.config_elasticsearch import SearchSettings
18+
from stac_fastapi.elasticsearch.database_logic.database_logic_elasticsearch import create_collection_index
19+
1220
from stac_fastapi.elasticsearch.extensions import QueryExtension
1321
from stac_fastapi.elasticsearch.session import Session
1422
from stac_fastapi.extensions.core import (
@@ -21,7 +29,7 @@
2129
)
2230
from stac_fastapi.extensions.third_party import BulkTransactionExtension
2331

24-
settings = ElasticsearchSettings()
32+
settings = SearchSettings()
2533
session = Session.create_from_settings(settings)
2634

2735
filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient())

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config/config_elasticsearch.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def _es_config() -> Dict[str, Any]:
4545
_forbidden_fields: Set[str] = {"type"}
4646

4747

48-
class ElasticsearchSettings(ApiSettings):
48+
class SearchSettings(ApiSettings):
4949
"""API settings."""
5050

5151
# Fields which are defined by STAC but not included in the database model
@@ -58,7 +58,7 @@ def create_client(self):
5858
return Elasticsearch(**_es_config())
5959

6060

61-
class AsyncElasticsearchSettings(ApiSettings):
61+
class AsyncSearchSettings(ApiSettings):
6262
"""API settings."""
6363

6464
# Fields which are defined by STAC but not included in the database model

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config/config_opensearch.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def _es_config() -> Dict[str, Any]:
1818
# Initialize the configuration dictionary
1919
config = {
2020
"hosts": hosts,
21-
"headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"},
21+
"headers": {"accept": "application/json", "Content-Type": "application/json"},
2222
}
2323

2424
# Explicitly exclude SSL settings when not using SSL
@@ -45,7 +45,7 @@ def _es_config() -> Dict[str, Any]:
4545
_forbidden_fields: Set[str] = {"type"}
4646

4747

48-
class ElasticsearchSettings(ApiSettings):
48+
class SearchSettings(ApiSettings):
4949
"""API settings."""
5050

5151
# Fields which are defined by STAC but not included in the database model
@@ -58,7 +58,7 @@ def create_client(self):
5858
return OpenSearch(**_es_config())
5959

6060

61-
class AsyncElasticsearchSettings(ApiSettings):
61+
class AsyncSearchSettings(ApiSettings):
6262
"""API settings."""
6363

6464
# Fields which are defined by STAC but not included in the database model

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Item crud client."""
22
import logging
33
import re
4+
import os
45
from base64 import urlsafe_b64encode
56
from datetime import datetime as datetime_type
67
from datetime import timezone
@@ -18,9 +19,16 @@
1819
from stac_pydantic.links import Relations
1920
from stac_pydantic.shared import MimeTypes
2021

22+
backend = os.getenv("BACKEND", "elasticsearch").lower()
23+
if backend == "opensearch":
24+
from stac_fastapi.elasticsearch.database_logic.database_logic_opensearch import DatabaseLogic
25+
from stac_fastapi.elasticsearch.config.config_opensearch import SearchSettings
26+
else:
27+
from stac_fastapi.elasticsearch.database_logic.database_logic_elasticsearch import DatabaseLogic
28+
from stac_fastapi.elasticsearch.config.config_elasticsearch import SearchSettings
29+
30+
2131
from stac_fastapi.elasticsearch import serializers
22-
from stac_fastapi.elasticsearch.config.config_elasticsearch import ElasticsearchSettings
23-
from stac_fastapi.elasticsearch.database_elasticsearch.database_logic.database_logic_elasticsearch import DatabaseLogic
2432
from stac_fastapi.elasticsearch.models.links import PagingLinks
2533
from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer
2634
from stac_fastapi.elasticsearch.session import Session
@@ -716,7 +724,7 @@ class BulkTransactionsClient(BaseBulkTransactionsClient):
716724

717725
def __attrs_post_init__(self):
718726
"""Create es engine."""
719-
settings = ElasticsearchSettings()
727+
settings = SearchSettings()
720728
self.client = settings.create_client
721729

722730
def preprocess_item(

0 commit comments

Comments
 (0)