Skip to content

Commit ae9b38f

Browse files
arcitecture, requirements, opensearch config
1 parent eb4af08 commit ae9b38f

File tree

14 files changed

+966
-48
lines changed

14 files changed

+966
-48
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
elasticsearch/snapshots/
22

3+
# local testing
4+
DEV.ipynb
5+
36
# Byte-compiled / optimized / DLL files
47
__pycache__/
58
*.py[cod]

README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,15 @@ docker-compose build
4242
docker-compose up
4343
```
4444

45-
By default, docker-compose uses Elasticsearch 8.x. However, most recent 7.x versions should also work.
45+
By default, docker-compose uses Elasticsearch 8.x and OpenSearch 2.11.1.
4646
If you wish to use a different version, put the following in a
4747
file named `.env` in the same directory you run docker-compose from:
4848

4949
```shell
5050
ELASTICSEARCH_VERSION=7.17.1
51+
OPENSEARCH_VERSION=2.11.0
5152
```
53+
The most recent 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information.
5254

5355
To create a new Collection:
5456

docker-compose.yml

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,33 @@
11
version: '3.9'
22

33
services:
4-
app-elasticsearch:
5-
container_name: stac-fastapi-es
6-
image: stac-utils/stac-fastapi-es
7-
restart: always
8-
build:
9-
context: .
10-
dockerfile: Dockerfile.dev
11-
environment:
12-
- APP_HOST=0.0.0.0
13-
- APP_PORT=8080
14-
- RELOAD=true
15-
- ENVIRONMENT=local
16-
- WEB_CONCURRENCY=10
17-
- ES_HOST=172.17.0.1
18-
- ES_PORT=9200
19-
- ES_USE_SSL=false
20-
- ES_VERIFY_CERTS=false
21-
ports:
22-
- "8080:8080"
23-
volumes:
24-
- ./stac_fastapi:/app/stac_fastapi
25-
- ./scripts:/app/scripts
26-
- ./esdata:/usr/share/elasticsearch/data
27-
depends_on:
28-
- elasticsearch
29-
command:
30-
bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app"
4+
# app-elasticsearch:
5+
# container_name: stac-fastapi-es
6+
# image: stac-utils/stac-fastapi-es
7+
# restart: always
8+
# build:
9+
# context: .
10+
# dockerfile: Dockerfile.dev
11+
# environment:
12+
# - APP_HOST=0.0.0.0
13+
# - APP_PORT=8080
14+
# - RELOAD=true
15+
# - ENVIRONMENT=local
16+
# - WEB_CONCURRENCY=10
17+
# - ES_HOST=172.17.0.1
18+
# - ES_PORT=9200
19+
# - ES_USE_SSL=false
20+
# - ES_VERIFY_CERTS=false
21+
# ports:
22+
# - "8080:8080"
23+
# volumes:
24+
# - ./stac_fastapi:/app/stac_fastapi
25+
# - ./scripts:/app/scripts
26+
# - ./esdata:/usr/share/elasticsearch/data
27+
# depends_on:
28+
# - elasticsearch
29+
# command:
30+
# bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app"
3131

3232
app-opensearch:
3333
container_name: stac-fastapi-os
@@ -57,20 +57,20 @@ services:
5757
command:
5858
bash -c "./scripts/wait-for-it-es.sh os-container:9200 && python -m stac_fastapi.elasticsearch.app"
5959

60-
elasticsearch:
61-
container_name: es-container
62-
image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
63-
environment:
64-
ES_JAVA_OPTS: -Xms512m -Xmx1g
65-
volumes:
66-
- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
67-
- ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots
68-
ports:
69-
- "9200:9200"
60+
# elasticsearch:
61+
# container_name: es-container
62+
# image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
63+
# environment:
64+
# ES_JAVA_OPTS: -Xms512m -Xmx1g
65+
# volumes:
66+
# - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
67+
# - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots
68+
# ports:
69+
# - "9200:9200"
7070

7171
opensearch:
7272
container_name: os-container
73-
image: opensearchproject/opensearch:latest
73+
image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-2.11.1}
7474
environment:
7575
- "discovery.type=single-node"
7676
- "plugins.security.disabled=true"

opensearch/config/opensearch.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
## Cluster Settings
22
cluster.name: stac-cluster
3-
node.name: es01
3+
node.name: os01
44
network.host: 0.0.0.0
55
transport.host: 0.0.0.0
66
discovery.type: single-node

stac_fastapi/elasticsearch/setup.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
"stac-fastapi.extensions==2.4.9",
1616
"elasticsearch[async]==8.11.0",
1717
"elasticsearch-dsl==8.11.0",
18+
"opensearch-py==2.1.1",
19+
"opensearch-py[async]==2.1.1",
1820
"pystac[validation]",
1921
"uvicorn",
2022
"orjson",

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
"""FastAPI application."""
22
from stac_fastapi.api.app import StacApi
33
from stac_fastapi.api.models import create_get_request_model, create_post_request_model
4-
from stac_fastapi.elasticsearch.config import ElasticsearchSettings
4+
from stac_fastapi.elasticsearch.config.config_elasticsearch import ElasticsearchSettings
55
from stac_fastapi.elasticsearch.core import (
66
BulkTransactionsClient,
77
CoreClient,
88
EsAsyncBaseFiltersClient,
99
TransactionsClient,
1010
)
11-
from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index
11+
from stac_fastapi.elasticsearch.database_elasticsearch.database_logic.database_logic_elasticsearch import create_collection_index
1212
from stac_fastapi.elasticsearch.extensions import QueryExtension
1313
from stac_fastapi.elasticsearch.session import Session
1414
from stac_fastapi.extensions.core import (
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""client config implementations."""
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
"""API configuration."""
2+
import os
3+
import ssl
4+
from typing import Any, Dict, Set
5+
6+
from opensearchpy import AsyncOpenSearch, OpenSearch
7+
from stac_fastapi.types.config import ApiSettings
8+
9+
10+
def _es_config() -> Dict[str, Any]:
11+
# Determine the scheme (http or https)
12+
use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true"
13+
scheme = "https" if use_ssl else "http"
14+
15+
# Configure the hosts parameter with the correct scheme
16+
hosts = [f"{scheme}://{os.getenv('ES_HOST')}:{os.getenv('ES_PORT')}"]
17+
18+
# Initialize the configuration dictionary
19+
config = {
20+
"hosts": hosts,
21+
"headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"},
22+
}
23+
24+
# Explicitly exclude SSL settings when not using SSL
25+
if not use_ssl:
26+
return config
27+
28+
# Include SSL settings if using https
29+
config["ssl_version"] = ssl.TLSVersion.TLSv1_3 # type: ignore
30+
config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore
31+
32+
# Include CA Certificates if verifying certs
33+
if config["verify_certs"]:
34+
config["ca_certs"] = os.getenv(
35+
"CURL_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt"
36+
)
37+
38+
# Handle authentication
39+
if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")):
40+
config["http_auth"] = (u, p)
41+
42+
return config
43+
44+
45+
_forbidden_fields: Set[str] = {"type"}
46+
47+
48+
class ElasticsearchSettings(ApiSettings):
49+
"""API settings."""
50+
51+
# Fields which are defined by STAC but not included in the database model
52+
forbidden_fields: Set[str] = _forbidden_fields
53+
indexed_fields: Set[str] = {"datetime"}
54+
55+
@property
56+
def create_client(self):
57+
"""Create es client."""
58+
return OpenSearch(**_es_config())
59+
60+
61+
class AsyncElasticsearchSettings(ApiSettings):
62+
"""API settings."""
63+
64+
# Fields which are defined by STAC but not included in the database model
65+
forbidden_fields: Set[str] = _forbidden_fields
66+
indexed_fields: Set[str] = {"datetime"}
67+
68+
@property
69+
def create_client(self):
70+
"""Create async elasticsearch client."""
71+
return AsyncOpenSearch(**_es_config())

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
from stac_pydantic.shared import MimeTypes
2020

2121
from stac_fastapi.elasticsearch import serializers
22-
from stac_fastapi.elasticsearch.config import ElasticsearchSettings
23-
from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import DatabaseLogic
22+
from stac_fastapi.elasticsearch.config.config_elasticsearch import ElasticsearchSettings
23+
from stac_fastapi.elasticsearch.database_elasticsearch.database_logic.database_logic_elasticsearch import DatabaseLogic
2424
from stac_fastapi.elasticsearch.models.links import PagingLinks
2525
from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer
2626
from stac_fastapi.elasticsearch.session import Session

0 commit comments

Comments
 (0)