feat(query): Enhanced Inverted Index for VARIANT Type to precise matching Object within Arrays

Cargo.toml

@@ -485,7 +485,7 @@ strum = "0.24.1"
 sub-cache = "0.2.1"
 sys-info = "0.9"
 sysinfo = "0.34.2"
-tantivy = "0.25.0"
+tantivy = "0.26.0"
 tantivy-common = "0.10.0"
 tantivy-fst = "0.5"
 tantivy-jieba = "0.17.0"
@@ -651,9 +651,9 @@ recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "1
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
 state-machine-api = { git = "https://github.com/databendlabs/state-machine-api.git", tag = "v0.3.4" }
 sub-cache = { git = "https://github.com/databendlabs/sub-cache", tag = "v0.2.1" }
-tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d" }
-tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d", package = "tantivy-common" }
-tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "ac27464" }
-tantivy-query-grammar = { git = "https://github.com/datafuse-extras/tantivy", rev = "9065a4d", package = "tantivy-query-grammar" }
+tantivy = { git = "https://github.com/b41sh/tantivy", rev = "7cec26400695dc9d8299751c3b36e8b9eee6abbc" }
+tantivy-common = { git = "https://github.com/b41sh/tantivy", rev = "7cec26400695dc9d8299751c3b36e8b9eee6abbc", package = "tantivy-common" }
+tantivy-jieba = { git = "https://github.com/b41sh/tantivy-jieba", rev = "0d18ff7872af581fb799c07f32cbad84ac5993f7" }
+tantivy-query-grammar = { git = "https://github.com/b41sh/tantivy", rev = "7cec26400695dc9d8299751c3b36e8b9eee6abbc", package = "tantivy-query-grammar" }
 watcher = { git = "https://github.com/databendlabs/watcher", tag = "v0.4.2" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }

src/query/service/tests/it/storages/testdata/columns_table.txt

@@ -500,3 +500,5 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo
 | 'webhook_options'                 | 'system'             | 'notifications'           | 'Nullable(Variant)'   | 'VARIANT'           | ''       | ''       | 'YES'    | ''       |
 | 'workload_groups'                 | 'system'             | 'users'                   | 'Nullable(String)'    | 'VARCHAR'           | ''       | ''       | 'YES'    | ''       |
 +-----------------------------------+----------------------+---------------------------+-----------------------+---------------------+----------+----------+----------+----------+
+
+

src/query/storages/fuse/src/io/read/inverted_index/inverted_index_reader.rs

@@ -175,7 +175,7 @@ impl InvertedIndexReader {
 
         let (matched_rows, matched_scores) = if self.has_score {
             let collector = TopDocs::with_limit(self.row_count as usize);
-            let docs = searcher.search(&query, &collector)?;
+            let docs = searcher.search(&query, &collector.order_by_score())?;
 
             let mut matched_rows = Vec::with_capacity(docs.len());
             let mut matched_scores = Vec::with_capacity(docs.len());
@@ -436,7 +436,7 @@ impl InvertedIndexReader {
                 block_postings_map.insert(term_id, block_postings);
             } else if slice_name.starts_with("pos") {
                 let term_id = id;
-                let position_reader = PositionReader::open(slice_data)?;
+                let position_reader = PositionReader::open(slice_data, None)?;
                 position_reader_map.insert(term_id, position_reader);
             } else if slice_name.starts_with("fieldnorm") {
                 let field_id = id as u32;

src/query/storages/fuse/src/pruning/inverted_index_pruner.rs

@@ -18,6 +18,7 @@ use std::sync::Arc;
 use databend_common_catalog::plan::InvertedIndexInfo;
 use databend_common_catalog::plan::PushDownInfo;
 use databend_common_exception::Result;
+use databend_common_expression::types::DataType;
 use databend_common_expression::types::F32;
 use databend_storages_common_io::ReadSettings;
 use opendal::Operator;
@@ -95,6 +96,14 @@ impl InvertedIndexPruner {
                     need_position = true;
                 }
             });
+            for field_id in &field_ids {
+                let field = inverted_index_info.index_schema.field(*field_id as usize);
+                let data_type = field.data_type().remove_nullable();
+                if data_type == DataType::Variant {
+                    need_position = true;
+                    break;
+                }
+            }
 
             // whether need to generate score internl column
             let has_score = inverted_index_info.has_score;

tests/sqllogictests/suites/query/index/04_inverted_index/04_0000_inverted_index_base.test

@@ -500,6 +500,47 @@ query IT
 select * from t2 where query('body:test');
 ----
 
+
+statement ok
+CREATE TABLE t3 (id int, body variant, INVERTED INDEX idx (body))
+
+statement ok
+INSERT INTO t3 VALUES
+(1, '{"videoInfo":{"extraData":[{ "name": "codecA", "type": "mp4" },{ "name": "codecB", "type": "jpg" }]}}'),
+(2, '{"videoInfo":{"extraData":[{ "name": "codecA", "type": "jpg" },{ "name": "codecA", "type": "mp4" }]}}'),
+(3, '{"videoInfo":{"extraData":[{ "name": "codecA", "type": "jpg" },{ "name": "codecB", "type": "mp4" }]}}'),
+(4, '{"videoInfo":{"extraData":[{ "name": "codecA", "attributes": { "type": "jpg" }}, { "name": "codecB", "attributes": { "type": "mp4" }}]}}'),
+(5, '{"videoInfo":{"extraData":[{ "name": "codecA", "attributes": { "type": "mp4" }}, { "name": "codecB", "attributes": { "type": "jpg" }}]}}'),
+(6, '{"videoInfo":{"extraData":[{ "name": "codec foo", "type": "jpg" }, { "name": "codec bar", "type": "mp4" }]}}'),
+(7, '{"videoInfo":{"extraData":[{ "name": "codec foo", "type": "mp4" }]}}');
+
+query IT
+select * from t3 where query('body.videoInfo.extraData.name:codecA AND body.videoInfo.extraData.type:jpg');
+----
+2 {"videoInfo":{"extraData":[{"name":"codecA","type":"jpg"},{"name":"codecA","type":"mp4"}]}}
+3 {"videoInfo":{"extraData":[{"name":"codecA","type":"jpg"},{"name":"codecB","type":"mp4"}]}}
+
+query IT
+select * from t3 where query('body.videoInfo.extraData.name:codecA AND body.videoInfo.extraData.type:mp4');
+----
+1 {"videoInfo":{"extraData":[{"name":"codecA","type":"mp4"},{"name":"codecB","type":"jpg"}]}}
+2 {"videoInfo":{"extraData":[{"name":"codecA","type":"jpg"},{"name":"codecA","type":"mp4"}]}}
+
+query IT
+select * from t3 where query('body.videoInfo.extraData.name:codecA AND body.videoInfo.extraData.attributes.type:jpg');
+----
+4 {"videoInfo":{"extraData":[{"attributes":{"type":"jpg"},"name":"codecA"},{"attributes":{"type":"mp4"},"name":"codecB"}]}}
+
+query IT
+select * from t3 where query('body.videoInfo.extraData.name:"codec foo" AND body.videoInfo.extraData.type:mp4');
+----
+7 {"videoInfo":{"extraData":[{"name":"codec foo","type":"mp4"}]}}
+
+query IT
+select * from t3 where query('body.videoInfo.extraData.name:codecB AND body.videoInfo.extraData.type:jpg');
+----
+1 {"videoInfo":{"extraData":[{"name":"codecA","type":"mp4"},{"name":"codecB","type":"jpg"}]}}
+
 statement ok
 CREATE TABLE t_native (id int, content string, INVERTED INDEX idx1 (content)) storage_format = 'native' row_per_page = 2;
 
@@ -528,3 +569,4 @@ use default
 statement ok
 drop database test_inverted_index
 
+