fix(query): fix query function parsing nested conditions (#18940)

* fix(query): fix query function parsing nested conditions

* fix

* fix

Author: b41sh

Cargo.lock

@@ -503,6 +503,7 @@ tantivy = "0.25.0"
 tantivy-common = "0.10.0"
 tantivy-fst = "0.5"
 tantivy-jieba = "0.17.0"
+tantivy-query-grammar = "0.25.0"
 temp-env = "0.3.0"
 tempfile = "3.4.0"
 terminal_size = "0.4.2"
@@ -660,5 +661,6 @@ sub-cache = { git = "https://github.com/databendlabs/sub-cache", tag = "v0.2.1"
 tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d" }
 tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d", package = "tantivy-common" }
 tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "ac27464" }
+tantivy-query-grammar = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d", package = "tantivy-query-grammar" }
 watcher = { git = "https://github.com/databendlabs/watcher", tag = "v0.4.2" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }

Cargo.toml

@@ -68,6 +68,7 @@ serde_json = { workspace = true }
 sha2 = { workspace = true }
 similar = { workspace = true }
 simsearch = { workspace = true }
+tantivy-query-grammar = { workspace = true }
 unicase = { workspace = true }
 url = { workspace = true }
 

src/query/sql/Cargo.toml

@@ -134,6 +134,9 @@ use jsonb::keypath::KeyPaths;
 use serde_json::json;
 use serde_json::to_string;
 use simsearch::SimSearch;
+use tantivy_query_grammar::parse_query_lenient;
+use tantivy_query_grammar::UserInputAst;
+use tantivy_query_grammar::UserInputLeaf;
 use unicase::Ascii;
 
 use super::name_resolution::NameResolutionContext;
@@ -2657,28 +2660,75 @@ impl<'a> TypeChecker<'a> {
             .set_span(query_scalar.span()));
         };
 
-        let field_strs: Vec<&str> = query_text.split(' ').collect();
-        let mut column_refs = Vec::with_capacity(field_strs.len());
-        for field_str in field_strs {
-            if !field_str.contains(':') {
-                continue;
+        // Extract the first subfield from the query field as the field name,
+        // as queries may contain dot separators when the field is JSON type.
+        // For example: The value of the `info` field is: `{“tags”:{“id”:10,“env”:“prod”,‘name’:“test”}}`
+        // The query statement can be written as `info.tags.env:prod`, the field `info` can be extracted.
+        fn extract_first_subfield(field: &str) -> String {
+            field.split('.').next().unwrap_or(field).to_string()
+        }
+
+        fn collect_fields(ast: &UserInputAst, fields: &mut HashSet<String>) {
+            match ast {
+                UserInputAst::Clause(clauses) => {
+                    for (_, sub_ast) in clauses {
+                        collect_fields(sub_ast, fields);
+                    }
+                }
+                UserInputAst::Boost(inner_ast, _) => {
+                    collect_fields(inner_ast, fields);
+                }
+                UserInputAst::Leaf(leaf) => match &**leaf {
+                    UserInputLeaf::Literal(literal) => {
+                        if let Some(field) = &literal.field_name {
+                            fields.insert(extract_first_subfield(field));
+                        }
+                    }
+                    UserInputLeaf::Range { field, .. } => {
+                        if let Some(field) = field {
+                            fields.insert(extract_first_subfield(field));
+                        }
+                    }
+                    UserInputLeaf::Set { field, .. } => {
+                        if let Some(field) = field {
+                            fields.insert(extract_first_subfield(field));
+                        }
+                    }
+                    UserInputLeaf::Exists { field } => {
+                        fields.insert(extract_first_subfield(field));
+                    }
+                    UserInputLeaf::Regex { field, .. } => {
+                        if let Some(field) = field {
+                            fields.insert(extract_first_subfield(field));
+                        }
+                    }
+                    UserInputLeaf::All => {}
+                },
             }
-            let field_names: Vec<&str> = field_str.split(':').collect();
-            // if the field is JSON type, must specify the key path in the object
-            // for example:
-            // the field `info` has the value: `{"tags":{"id":10,"env":"prod","name":"test"}}`
-            // a query can be written like this `info.tags.env:prod`
-            let field_name = field_names[0].trim();
-            let sub_field_names: Vec<&str> = field_name.split('.').collect();
+        }
+
+        let (query_ast, errs) = parse_query_lenient(query_text);
+        if !errs.is_empty() {
+            let err_msg = errs
+                .into_iter()
+                .map(|err| format!("{} pos {}", err.message, err.pos))
+                .join(", ");
+            return Err(
+                ErrorCode::SemanticError(format!("invalid query: {err_msg}",))
+                    .set_span(query_scalar.span()),
+            );
+        }
+        let mut fields = HashSet::new();
+        collect_fields(&query_ast, &mut fields);
+
+        let mut column_refs = Vec::with_capacity(fields.len());
+        for field in fields.into_iter() {
             let column_expr = Expr::ColumnRef {
                 span: query_scalar.span(),
                 column: ColumnRef {
                     database: None,
                     table: None,
-                    column: ColumnID::Name(Identifier::from_name(
-                        query_scalar.span(),
-                        sub_field_names[0].trim(),
-                    )),
+                    column: ColumnID::Name(Identifier::from_name(query_scalar.span(), field)),
                 },
             };
             let box (field_scalar, _) = self.resolve(&column_expr)?;