zhiqiang-hhhh
diff --git a/‎be/CMakeLists.txt‎
Lines changed: 10 additions & 1 deletion b/‎be/CMakeLists.txt‎
Lines changed: 10 additions & 1 deletion
diff --git a/‎be/src/apache-orc‎ b/‎be/src/apache-orc‎
diff --git a/‎be/src/cloud/CMakeLists.txt‎
Lines changed: 1 addition & 0 deletions b/‎be/src/cloud/CMakeLists.txt‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎be/src/clucene‎ b/‎be/src/clucene‎
diff --git a/‎be/src/common/consts.h‎
Lines changed: 1 addition & 0 deletions b/‎be/src/common/consts.h‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎be/src/common/status.h‎
Lines changed: 1 addition & 1 deletion b/‎be/src/common/status.h‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎be/src/index-tools/index_tool.cpp‎
Lines changed: 13 additions & 13 deletions b/‎be/src/index-tools/index_tool.cpp‎
Lines changed: 13 additions & 13 deletions
diff --git a/‎be/src/olap/accept_null_predicate.h‎
Lines changed: 1 addition & 1 deletion b/‎be/src/olap/accept_null_predicate.h‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎be/src/olap/column_predicate.h‎
Lines changed: 2 additions & 2 deletions b/‎be/src/olap/column_predicate.h‎
Lines changed: 2 additions & 2 deletions
diff --git a/‎be/src/olap/compaction.cpp‎
Lines changed: 24 additions & 27 deletions b/‎be/src/olap/compaction.cpp‎
Lines changed: 24 additions & 27 deletions
@@ -177,7 +177,7 @@ endif()
 
 set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
 
-option(BUILD_FAISS "Link doris with faiss for vector similarity search" OFF)
+option(BUILD_FAISS "Link doris with faiss for vector similarity search" ON)
 message(STATUS "build faiss: ${BUILD_FAISS}")
 
 include (cmake/thirdparty.cmake)
@@ -283,6 +283,7 @@ add_compile_options(-g
 
 add_compile_options(-Wno-unused-parameter
                     -Wno-deprecated-declarations
+                    -Wno-self-assign-field
                     -Wno-sign-compare)
 
 if (COMPILER_GCC)
@@ -382,6 +383,10 @@ if (USE_DWARF)
     add_compile_options(-gdwarf-5)
 endif()
 
+if (BUILD_FAISS)
+    add_definitions(-DBUILD_FAISS)
+endif()
+
 # For CMAKE_BUILD_TYPE=Debug
 if (OS_MACOSX AND ARCH_ARM)
     # Using -O0 may meet ARM64 branch out of range errors when linking with tcmalloc.
@@ -519,6 +524,7 @@ set(DORIS_LINK_LIBS
     Cloud
     ${WL_END_GROUP}
     CommonCPP
+    vector
 )
 
 set(absl_DIR ${THIRDPARTY_DIR}/lib/cmake/absl)
@@ -760,6 +766,7 @@ function(pch_reuse target)
     endif()
 endfunction(pch_reuse target)
 
+
 add_subdirectory(${SRC_DIR}/agent)
 add_subdirectory(${SRC_DIR}/common)
 add_subdirectory(${SRC_DIR}/exec)
@@ -775,6 +782,8 @@ add_subdirectory(${SRC_DIR}/runtime_filter)
 add_subdirectory(${SRC_DIR}/service) # this include doris_be
 add_subdirectory(${SRC_DIR}/udf)
 add_subdirectory(${SRC_DIR}/cloud)
+add_subdirectory(${SRC_DIR}/vector)
+
 
 option(BUILD_META_TOOL "Build meta tool" OFF)
 if (BUILD_META_TOOL)
 
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+
 # where to put generated libraries
 set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/cloud")
 
 
@@ -31,6 +31,7 @@ const std::string GLOBAL_ROWID_COL = "__DORIS_GLOBAL_ROWID_COL__";
 const std::string ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
 const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
 const std::string PARTIAL_UPDATE_AUTO_INC_COL = "__PARTIAL_UPDATE_AUTO_INC_COLUMN__";
+const std::string VIRTUAL_COLUMN_PREFIX = "__DORIS_VIRTUAL_COL__";
 
 /// The maximum precision representable by a 4-byte decimal (Decimal4Value)
 constexpr int MAX_DECIMAL32_PRECISION = 9;
 
@@ -278,7 +278,7 @@ namespace ErrorCode {
     E(SEGCOMPACTION_FAILED, -3119, false);                   \
     E(ROWSET_ADD_TO_BINLOG_FAILED, -3122, true);             \
     E(ROWSET_BINLOG_NOT_ONLY_ONE_VERSION, -3123, true);      \
-    E(INVERTED_INDEX_INVALID_PARAMETERS, -6000, false);      \
+    E(INDEX_INVALID_PARAMETERS, -6000, false);               \
     E(INVERTED_INDEX_NOT_SUPPORTED, -6001, false);           \
     E(INVERTED_INDEX_CLUCENE_ERROR, -6002, false);           \
     E(INVERTED_INDEX_FILE_NOT_FOUND, -6003, false);          \
 
@@ -41,19 +41,19 @@
 #pragma clang diagnostic pop
 #endif
 #include "io/fs/local_file_system.h"
+#include "olap/rowset/segment_v2/index_file_reader.h"
+#include "olap/rowset/segment_v2/index_file_writer.h"
 #include "olap/rowset/segment_v2/inverted_index/query/conjunction_query.h"
 #include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
 #include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
-#include "olap/rowset/segment_v2/inverted_index_file_writer.h"
 #include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
 #include "olap/tablet_schema.h"
 
 using doris::segment_v2::DorisCompoundReader;
 using doris::segment_v2::DorisFSDirectoryFactory;
-using doris::segment_v2::InvertedIndexFileWriter;
+using doris::segment_v2::IndexFileWriter;
 using doris::segment_v2::InvertedIndexDescriptor;
-using doris::segment_v2::InvertedIndexFileReader;
+using doris::segment_v2::IndexFileReader;
 using doris::io::FileInfo;
 using doris::TabletIndex;
 using namespace doris::segment_v2;
@@ -550,14 +550,14 @@ int main(int argc, char** argv) {
         }
 
         auto fs = doris::io::global_local_filesystem();
-        auto index_file_writer = std::make_unique<InvertedIndexFileWriter>(
+        auto index_file_writer = std::make_unique<IndexFileWriter>(
                 fs,
                 std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
                         doris::local_segment_path(file_dir, rowset_id, seg_id))},
                 rowset_id, seg_id, doris::InvertedIndexStorageFormatPB::V2);
         auto st = index_file_writer->open(&index_meta);
         if (!st.has_value()) {
-            std::cerr << "InvertedIndexFileWriter init error:" << st.error() << std::endl;
+            std::cerr << "IndexFileWriter init error:" << st.error() << std::endl;
             return -1;
         }
         using T = std::decay_t<decltype(st)>;
@@ -601,7 +601,7 @@ int main(int argc, char** argv) {
 
         auto ret = index_file_writer->close();
         if (!ret.ok()) {
-            std::cerr << "InvertedIndexFileWriter close error:" << ret.msg() << std::endl;
+            std::cerr << "IndexFileWriter close error:" << ret.msg() << std::endl;
             return -1;
         }
     } else if (FLAGS_operation == "show_nested_files_v2") {
@@ -613,11 +613,11 @@ int main(int argc, char** argv) {
                 google::protobuf::StripSuffixString(FLAGS_idx_file_path, ".idx");
         auto fs = doris::io::global_local_filesystem();
         try {
-            auto index_file_reader = std::make_unique<InvertedIndexFileReader>(
+            auto index_file_reader = std::make_unique<IndexFileReader>(
                     fs, index_path_prefix, doris::InvertedIndexStorageFormatPB::V2);
             auto st = index_file_reader->init(4096);
             if (!st.ok()) {
-                std::cerr << "InvertedIndexFileReader init error:" << st.msg() << std::endl;
+                std::cerr << "IndexFileReader init error:" << st.msg() << std::endl;
                 return -1;
             }
             std::cout << "Nested files for " << index_path_prefix << std::endl;
@@ -638,7 +638,7 @@ int main(int argc, char** argv) {
                 CLuceneError err;
                 auto ret = index_file_reader->open(&index_meta);
                 if (!ret.has_value()) {
-                    std::cerr << "InvertedIndexFileReader open error:" << ret.error() << std::endl;
+                    std::cerr << "IndexFileReader open error:" << ret.error() << std::endl;
                     return -1;
                 }
                 using T = std::decay_t<decltype(ret)>;
@@ -660,11 +660,11 @@ int main(int argc, char** argv) {
                 google::protobuf::StripSuffixString(FLAGS_idx_file_path, ".idx");
         auto fs = doris::io::global_local_filesystem();
         try {
-            auto index_file_reader = std::make_unique<InvertedIndexFileReader>(
+            auto index_file_reader = std::make_unique<IndexFileReader>(
                     fs, index_path_prefix, doris::InvertedIndexStorageFormatPB::V2);
             auto st = index_file_reader->init(4096);
             if (!st.ok()) {
-                std::cerr << "InvertedIndexFileReader init error:" << st.msg() << std::endl;
+                std::cerr << "IndexFileReader init error:" << st.msg() << std::endl;
                 return -1;
             }
             std::vector<std::string> files;
@@ -677,7 +677,7 @@ int main(int argc, char** argv) {
             index_meta.init_from_pb(index_pb);
             auto ret = index_file_reader->open(&index_meta);
             if (!ret.has_value()) {
-                std::cerr << "InvertedIndexFileReader open error:" << ret.error() << std::endl;
+                std::cerr << "IndexFileReader open error:" << ret.error() << std::endl;
                 return -1;
             }
             using T = std::decay_t<decltype(ret)>;
 
@@ -51,7 +51,7 @@ class AcceptNullPredicate : public ColumnPredicate {
     }
 
     Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
-                    InvertedIndexIterator* iterator, uint32_t num_rows,
+                    IndexIterator* iterator, uint32_t num_rows,
                     roaring::Roaring* bitmap) const override {
         return _nested->evaluate(name_with_type, iterator, num_rows, bitmap);
     }
 
@@ -23,7 +23,7 @@
 #include "common/exception.h"
 #include "olap/rowset/segment_v2/bitmap_index_reader.h"
 #include "olap/rowset/segment_v2/bloom_filter.h"
-#include "olap/rowset/segment_v2/inverted_index_reader.h"
+#include "olap/rowset/segment_v2/inverted_index_iterator.h"
 #include "runtime/define_primitive_type.h"
 #include "util/runtime_profile.h"
 #include "vec/columns/column.h"
@@ -170,7 +170,7 @@ class ColumnPredicate {
 
     //evaluate predicate on inverted
     virtual Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
-                            InvertedIndexIterator* iterator, uint32_t num_rows,
+                            IndexIterator* iterator, uint32_t num_rows,
                             roaring::Roaring* bitmap) const {
         return Status::NotSupported(
                 "Not Implemented evaluate with inverted index, please check the predicate");
 
@@ -60,11 +60,12 @@
 #include "olap/rowset/rowset_meta.h"
 #include "olap/rowset/rowset_writer.h"
 #include "olap/rowset/rowset_writer_context.h"
+#include "olap/rowset/segment_v2/index_file_reader.h"
+#include "olap/rowset/segment_v2/index_file_writer.h"
 #include "olap/rowset/segment_v2/inverted_index_compaction.h"
 #include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
-#include "olap/rowset/segment_v2/inverted_index_file_writer.h"
 #include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/tmp_file_dirs.h"
 #include "olap/storage_engine.h"
 #include "olap/storage_policy.h"
 #include "olap/tablet.h"
@@ -671,8 +672,7 @@ Status Compaction::do_inverted_index_compaction() {
     }
 
     // src index dirs
-    std::vector<std::unique_ptr<InvertedIndexFileReader>> inverted_index_file_readers(
-            src_segment_num);
+    std::vector<std::unique_ptr<IndexFileReader>> index_file_readers(src_segment_num);
     for (const auto& m : src_seg_to_id_map) {
         const auto& [rowset_id, seg_id] = m.first;
 
@@ -713,12 +713,12 @@ Status Compaction::do_inverted_index_compaction() {
                     "get segment path failed. tablet_id={} rowset_id={} seg_id={}",
                     _tablet->tablet_id(), rowset_id.to_string(), seg_id);
         }
-        auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
+        auto index_file_reader = std::make_unique<IndexFileReader>(
                 fs,
                 std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value())},
                 _cur_tablet_schema->get_inverted_index_storage_format(),
                 rowset->rowset_meta()->inverted_index_file_info(seg_id));
-        auto st = inverted_index_file_reader->init(config::inverted_index_read_buffer_size);
+        auto st = index_file_reader->init(config::inverted_index_read_buffer_size);
         DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_init_inverted_index_file_reader",
                         {
                             st = Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
@@ -736,26 +736,24 @@ Status Compaction::do_inverted_index_compaction() {
                     "init inverted index file reader failed. tablet_id={} rowset_id={} seg_id={}",
                     _tablet->tablet_id(), rowset_id.to_string(), seg_id);
         }
-        inverted_index_file_readers[m.second] = std::move(inverted_index_file_reader);
+        index_file_readers[m.second] = std::move(index_file_reader);
     }
 
     // dest index files
     // format: rowsetId_segmentId
-    auto& inverted_index_file_writers = dynamic_cast<BaseBetaRowsetWriter*>(_output_rs_writer.get())
-                                                ->inverted_index_file_writers();
-    DBUG_EXECUTE_IF(
-            "Compaction::do_inverted_index_compaction_inverted_index_file_writers_size_error",
-            { inverted_index_file_writers.clear(); })
-    if (inverted_index_file_writers.size() != dest_segment_num) {
+    auto& index_file_writers =
+            dynamic_cast<BaseBetaRowsetWriter*>(_output_rs_writer.get())->index_file_writers();
+    DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_index_file_writers_size_error",
+                    { index_file_writers.clear(); })
+    if (index_file_writers.size() != dest_segment_num) {
         LOG(WARNING) << "failed to do index compaction, dest segment num not match. tablet_id="
                      << _tablet->tablet_id() << " dest_segment_num=" << dest_segment_num
-                     << " inverted_index_file_writers.size()="
-                     << inverted_index_file_writers.size();
+                     << " index_file_writers.size()=" << index_file_writers.size();
         mark_skip_index_compaction(ctx, error_handler);
         return Status::Error<INVERTED_INDEX_COMPACTION_ERROR>(
                 "dest segment num not match. tablet_id={} dest_segment_num={} "
-                "inverted_index_file_writers.size()={}",
-                _tablet->tablet_id(), dest_segment_num, inverted_index_file_writers.size());
+                "index_file_writers.size()={}",
+                _tablet->tablet_id(), dest_segment_num, index_file_writers.size());
     }
 
     // use tmp file dir to store index files
@@ -785,7 +783,7 @@ Status Compaction::do_inverted_index_compaction() {
         try {
             std::vector<std::unique_ptr<DorisCompoundReader>> src_idx_dirs(src_segment_num);
             for (int src_segment_id = 0; src_segment_id < src_segment_num; src_segment_id++) {
-                auto res = inverted_index_file_readers[src_segment_id]->open(index_meta);
+                auto res = index_file_readers[src_segment_id]->open(index_meta);
                 DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_reader", {
                     res = ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                             "debug point: Compaction::open_index_file_reader error"));
@@ -801,10 +799,10 @@ Status Compaction::do_inverted_index_compaction() {
                 src_idx_dirs[src_segment_id] = std::move(res.value());
             }
             for (int dest_segment_id = 0; dest_segment_id < dest_segment_num; dest_segment_id++) {
-                auto res = inverted_index_file_writers[dest_segment_id]->open(index_meta);
-                DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_writer", {
+                auto res = index_file_writers[dest_segment_id]->open(index_meta);
+                DBUG_EXECUTE_IF("Compaction::open_index_file_writer", {
                     res = ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
-                            "debug point: Compaction::open_inverted_index_file_writer error"));
+                            "debug point: Compaction::open_index_file_writer error"));
                 })
                 if (!res.has_value()) {
                     LOG(WARNING) << "failed to do index compaction, open inverted index file "
@@ -815,7 +813,7 @@ Status Compaction::do_inverted_index_compaction() {
                     throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, res.error().msg());
                 }
                 // Destination directories in dest_index_dirs do not need to be deconstructed,
-                // but their lifecycle must be managed by inverted_index_file_writers.
+                // but their lifecycle must be managed by index_file_writers.
                 dest_index_dirs[dest_segment_id] = res.value().get();
             }
             auto st = compact_column(index_meta->index_id(), src_idx_dirs, dest_index_dirs,
@@ -953,15 +951,14 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
 
                 std::string index_file_path;
                 try {
-                    auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
+                    auto index_file_reader = std::make_unique<IndexFileReader>(
                             fs,
                             std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
                                     seg_path.value())},
                             _cur_tablet_schema->get_inverted_index_storage_format(),
                             rowset->rowset_meta()->inverted_index_file_info(i));
-                    auto st = inverted_index_file_reader->init(
-                            config::inverted_index_read_buffer_size);
-                    index_file_path = inverted_index_file_reader->get_index_file_path(index_meta);
+                    auto st = index_file_reader->init(config::inverted_index_read_buffer_size);
+                    index_file_path = index_file_reader->get_index_file_path(index_meta);
                     DBUG_EXECUTE_IF(
                             "Compaction::construct_skip_inverted_index_index_file_reader_init_"
                             "status_not_ok",
@@ -978,7 +975,7 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
                     }
 
                     // check index meta
-                    auto result = inverted_index_file_reader->open(index_meta);
+                    auto result = index_file_reader->open(index_meta);
                     DBUG_EXECUTE_IF(
                             "Compaction::construct_skip_inverted_index_index_file_reader_open_"
                             "error",
Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ class AcceptNullPredicate : public ColumnPredicate {`
`51`	`51`	`}`
`52`	`52`
`53`	`53`	`Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,`
`54`		`- InvertedIndexIterator* iterator, uint32_t num_rows,`
	`54`	`+ IndexIterator* iterator, uint32_t num_rows,`
`55`	`55`	`roaring::Roaring* bitmap) const override {`
`56`	`56`	`return _nested->evaluate(name_with_type, iterator, num_rows, bitmap);`
`57`	`57`	`}`