update tantivy (#5892)

* update tantivy

* include columnar compression, fix build

* fix build

---------

Co-authored-by: PSeitz <PSeitz@users.noreply.github.com>

Author: PSeitz-dd

quickwit/Cargo.lock

@@ -346,11 +346,12 @@ quickwit-serve = { path = "quickwit-serve" }
 quickwit-storage = { path = "quickwit-storage" }
 quickwit-telemetry = { path = "quickwit-telemetry" }
 
-tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "80f5f1e", default-features = false, features = [
+tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "7963b0b", default-features = false, features = [
   "lz4-compression",
   "mmap",
   "quickwit",
   "zstd-compression",
+  "columnar-zstd-compression",
 ] }
 tantivy-fst = "0.5"
 

quickwit/Cargo.toml

@@ -360,8 +360,8 @@ fn field_metadata_to_list_field_serialized(
     ListFieldsEntryResponse {
         field_name: field_metadata.field_name.to_string(),
         field_type: tantivy_type_to_list_field_type(field_metadata.typ) as i32,
-        searchable: field_metadata.indexed,
-        aggregatable: field_metadata.fast,
+        searchable: field_metadata.is_indexed(),
+        aggregatable: field_metadata.is_fast(),
         index_ids: Vec::new(),
         non_searchable_index_ids: Vec::new(),
         non_aggregatable_index_ids: Vec::new(),
@@ -398,23 +398,29 @@ mod tests {
             FieldMetadata {
                 field_name: "test".to_string(),
                 typ: Type::Str,
-                indexed: true,
                 stored: true,
-                fast: true,
+                fast_size: Some(10u64.into()),
+                term_dictionary_size: Some(10u64.into()),
+                postings_size: Some(10u64.into()),
+                positions_size: Some(10u64.into()),
             },
             FieldMetadata {
                 field_name: "test2".to_string(),
                 typ: Type::Str,
-                indexed: true,
                 stored: false,
-                fast: false,
+                fast_size: None,
+                term_dictionary_size: Some(10u64.into()),
+                postings_size: Some(10u64.into()),
+                positions_size: Some(10u64.into()),
             },
             FieldMetadata {
                 field_name: "test3".to_string(),
                 typ: Type::U64,
-                indexed: true,
                 stored: false,
-                fast: true,
+                fast_size: Some(10u64.into()),
+                term_dictionary_size: Some(10u64.into()),
+                postings_size: Some(10u64.into()),
+                positions_size: Some(10u64.into()),
             },
         ];
 

quickwit/quickwit-indexing/src/actors/packager.rs

@@ -178,6 +178,7 @@ fn convert_user_input_ast_to_query_ast(
                 Ok(term_set_query.into())
             }
             UserInputLeaf::Exists { field } => Ok(FieldPresenceQuery { field }.into()),
+            UserInputLeaf::Regex { .. } => bail!("regex query is not supported"),
         },
         UserInputAst::Boost(underlying, boost) => {
             let query_ast = convert_user_input_ast_to_query_ast(

quickwit/quickwit-query/src/query_ast/user_input_query.rs

@@ -195,7 +195,7 @@ fn compute_tantivy_ast_query_for_json(
 ) -> Result<TantivyQueryAst, InvalidQuery> {
     let mut bool_query = TantivyBoolQuery::default();
     let term = Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled());
-    if let Some(term) = convert_to_fast_value_and_append_to_json_term(term, text, true) {
+    if let Some(term) = convert_to_fast_value_and_append_to_json_term(&term, text, true) {
         bool_query
             .should
             .push(TantivyTermQuery::new(term, IndexRecordOption::Basic).into());