LIKE wildcard search

Author: jamesfisher-geo

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py

@@ -19,11 +19,15 @@
     TokenPaginationExtension,
     TransactionExtension,
 )
+
 from stac_fastapi.extensions.third_party import BulkTransactionExtension
 
 settings = ElasticsearchSettings()
 session = Session.create_from_settings(settings)
 
+filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient())
+filter_extension.conformance_classes.append("http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators")
+
 extensions = [
     TransactionExtension(client=TransactionsClient(session=session), settings=settings),
     BulkTransactionExtension(client=BulkTransactionsClient(session=session)),
@@ -32,7 +36,7 @@
     SortExtension(),
     TokenPaginationExtension(),
     ContextExtension(),
-    FilterExtension(client=EsAsyncBaseFiltersClient()),
+    filter_extension
 ]
 
 post_request_model = create_post_request_model(extensions)

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py

@@ -78,6 +78,15 @@ def to_es(self):
             )
 
 
+class AdvancedComparisonOp(str, Enum):
+    """Advanced Comparison operator.
+
+    CQL2 advanced comparison operator like (~).
+    """
+
+    like = "like"
+
+
 class SpatialIntersectsOp(str, Enum):
     """Spatial intersections operator s_intersects."""
 
@@ -152,7 +161,7 @@ def validate(cls, v):
 class Clause(BaseModel):
     """Filter extension clause."""
 
-    op: Union[LogicalOp, ComparisonOp, SpatialIntersectsOp]
+    op: Union[LogicalOp, ComparisonOp, AdvancedComparisonOp, SpatialIntersectsOp]
     args: List[Arg]
 
     def to_es(self):
@@ -171,6 +180,16 @@ def to_es(self):
                     "must_not": [{"term": {to_es(self.args[0]): to_es(self.args[1])}}]
                 }
             }
+        elif self.op == AdvancedComparisonOp.like:
+           return {
+               "wildcard": {
+                   to_es(self.args[0]): {
+                       "value": cql2_like_to_es(str(to_es(self.args[1]))),
+                        "boost": 1.0,
+                        "case_insensitive": "true"
+                    }
+                }
+            }
         elif (
             self.op == ComparisonOp.lt
             or self.op == ComparisonOp.lte
@@ -210,3 +229,25 @@ def to_es(arg: Arg):
         return arg
     else:
         raise RuntimeError(f"unknown arg {repr(arg)}")
+    
+
+def cql2_like_to_es(input_string):
+    """
+    Convert arugument in CQL2 ('_' and '%') to Elasticsearch wildcard operators ('?' and '*', respectively). Handle escape characters and 
+    handle Elasticsearch wildcards directly.
+    """
+    es_string = ""
+    escape = False
+
+    for char in input_string:
+        if char == "\\":
+            escape = True
+        elif char == '_' and not escape:
+            es_string += '?'
+        elif char == '%' and not escape:
+            es_string += '*'
+        else:
+            es_string += char
+            escape = False
+
+    return es_string

stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json

@@ -0,0 +1,9 @@
+{
+  "op": "like",
+  "args": [
+    {
+      "property": "scene_id"
+    },
+    "LC82030282019133%"
+  ]
+}

stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json

@@ -0,0 +1,10 @@
+{
+    "op": "like",
+    "args": [
+      {
+        "property": "scene_id"
+      },
+      "LC82030282019133LGN0_"
+    ]
+  }
+  

stac_fastapi/elasticsearch/tests/extensions/test_filter.py

@@ -213,3 +213,97 @@ async def test_search_filter_extension_floats_post(app_client, ctx):
 
     assert resp.status_code == 200
     assert len(resp.json()["features"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_search_filter_extension_wildcard_cql2(app_client, ctx):
+    single_char = ctx.item["id"][:-1] + "_"
+    multi_char = ctx.item["id"][:-3] + "%"
+
+    params = {
+        "filter": {
+            "op": "and",
+            "args": [
+                {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]},
+                {
+                    "op": "like",
+                    "args": [
+                        {"property": "id"},
+                        single_char,
+                    ],
+                },
+                {
+                    "op": "like",
+                    "args": [
+                        {"property": "id"},
+                        multi_char,
+                    ],
+                },
+            ],
+        }
+    }
+
+    resp = await app_client.post("/search", json=params)
+
+    assert resp.status_code == 200
+    assert len(resp.json()["features"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_search_filter_extension_wildcard_es(app_client, ctx):
+    single_char = ctx.item["id"][:-1] + "?"
+    multi_char = ctx.item["id"][:-3] + "*"
+
+    params = {
+        "filter": {
+            "op": "and",
+            "args": [
+                {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]},
+                {
+                    "op": "like",
+                    "args": [
+                        {"property": "id"},
+                        single_char,
+                    ],
+                },
+                {
+                    "op": "like",
+                    "args": [
+                        {"property": "id"},
+                        multi_char,
+                    ],
+                },
+            ],
+        }
+    }
+
+    resp = await app_client.post("/search", json=params)
+
+    assert resp.status_code == 200
+    assert len(resp.json()["features"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_search_filter_extension_escape_chars(app_client, ctx):
+    esc_chars = ctx.item["properties"]["landsat:product_id"].replace("_", "\_")[:-1] + "_"
+
+    params = {
+        "filter": {
+            "op": "and",
+            "args": [
+                {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]},
+                {
+                    "op": "like",
+                    "args": [
+                        {"property": "properties.landsat:product_id"},
+                        esc_chars,
+                    ],
+                }
+            ],
+        }
+    }
+
+    resp = await app_client.post("/search", json=params)
+
+    assert resp.status_code == 200
+    assert len(resp.json()["features"]) == 1