[fix](inverted index) fix index memeory leak for inverted index

be/src/olap/rowset/segment_v2/index_storage_format_v2.cpp

@@ -187,8 +187,7 @@ IndexStorageFormatV2::create_output_stream() {
 
     DCHECK(_index_file_writer->_idx_v2_writer != nullptr)
             << "inverted index file writer v2 is nullptr";
-    auto compound_file_output = std::unique_ptr<lucene::store::IndexOutput>(
-            out_dir->createOutputV2(_index_file_writer->_idx_v2_writer.get()));
+    auto compound_file_output = out_dir->createOutputV2(_index_file_writer->_idx_v2_writer.get());
 
     return {std::move(out_dir_ptr), std::move(compound_file_output)};
 }

be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp

@@ -693,9 +693,23 @@ lucene::store::IndexOutput* DorisFSDirectory::createOutput(const char* name) {
     return ret;
 }
 
-lucene::store::IndexOutput* DorisFSDirectory::createOutputV2(io::FileWriter* file_writer) {
-    auto* ret = _CLNEW FSIndexOutputV2();
-    ret->init(file_writer);
+std::unique_ptr<lucene::store::IndexOutput> DorisFSDirectory::createOutputV2(
+        io::FileWriter* file_writer) {
+    auto ret = std::make_unique<FSIndexOutputV2>();
+    ErrorContext error_context;
+    try {
+        ret->init(file_writer);
+    } catch (CLuceneError& err) {
+        error_context.eptr = std::current_exception();
+        error_context.err_msg.append("FSIndexOutputV2 init error: ");
+        error_context.err_msg.append(err.what());
+        LOG(ERROR) << error_context.err_msg;
+    }
+    FINALLY_EXCEPTION({
+        if (error_context.eptr) {
+            FINALLY_CLOSE(ret);
+        }
+    })
     return ret;
 }
 

be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h

@@ -79,7 +79,7 @@ class CLUCENE_EXPORT DorisFSDirectory : public lucene::store::Directory {
     void renameFile(const char* from, const char* to) override;
     void touchFile(const char* name) override;
     lucene::store::IndexOutput* createOutput(const char* name) override;
-    lucene::store::IndexOutput* createOutputV2(io::FileWriter* file_writer);
+    std::unique_ptr<lucene::store::IndexOutput> createOutputV2(io::FileWriter* file_writer);
     void close() override;
     std::string toString() const override;
     static const char* getClassName();

be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp

@@ -21,6 +21,7 @@
 #include <CLucene/util/bkd/bkd_reader.h>
 
 #include "common/config.h"
+#include "olap/rowset/segment_v2/inverted_index_common.h"
 #include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
 #include "olap/rowset/segment_v2/inverted_index_desc.h"
 #include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
@@ -29,10 +30,10 @@ namespace doris::segment_v2 {
 Status FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory,
                                            OptionalIndexSearcherPtr& output_searcher) {
     auto close_directory = true;
-    lucene::index::IndexReader* reader = nullptr;
+    std::unique_ptr<lucene::index::IndexReader> reader;
     try {
-        reader = lucene::index::IndexReader::open(
-                directory, config::inverted_index_read_buffer_size, close_directory);
+        reader = std::unique_ptr<lucene::index::IndexReader>(lucene::index::IndexReader::open(
+                directory, config::inverted_index_read_buffer_size, close_directory));
     } catch (const CLuceneError& e) {
         std::vector<std::string> file_names;
         directory->list(&file_names);
@@ -44,16 +45,15 @@ Status FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory,
         return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(msg);
     }
     bool close_reader = true;
-    auto index_searcher = std::make_shared<lucene::search::IndexSearcher>(reader, close_reader);
+    reader_size = reader->getTermInfosRAMUsed();
+    auto index_searcher =
+            std::make_shared<lucene::search::IndexSearcher>(reader.release(), close_reader);
     if (!index_searcher) {
         output_searcher = std::nullopt;
         return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                 "FulltextIndexSearcherBuilder build index_searcher error.");
     }
-    reader_size = reader->getTermInfosRAMUsed();
-    // NOTE: need to cl_refcount-- here, so that directory will be deleted when
-    // index_searcher is destroyed
-    _CLDECDELETE(directory)
+    // NOTE: IndexSearcher takes ownership of the reader, and directory cleanup is handled by caller
     output_searcher = index_searcher;
     return Status::OK();
 }
@@ -69,7 +69,6 @@ Status BKDIndexSearcherBuilder::build(lucene::store::Directory* directory,
         }
         reader_size = bkd_reader->ram_bytes_used();
         output_searcher = bkd_reader;
-        _CLDECDELETE(directory)
         return Status::OK();
     } catch (const CLuceneError& e) {
         return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
@@ -104,13 +103,13 @@ Result<std::unique_ptr<IndexSearcherBuilder>> IndexSearcherBuilder::create_index
 Result<IndexSearcherPtr> IndexSearcherBuilder::get_index_searcher(
         lucene::store::Directory* directory) {
     OptionalIndexSearcherPtr result;
-    auto st = build(directory, result);
+    std::unique_ptr<lucene::store::Directory, DirectoryDeleter> directory_ptr(directory);
+
+    auto st = build(directory_ptr.get(), result);
     if (!st.ok()) {
-        _CLDECDELETE(directory)
         return ResultError(st);
     }
     if (!result.has_value()) {
-        _CLDECDELETE(directory)
         return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                 "InvertedIndexSearcherCache build error."));
     }

be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp

@@ -556,7 +556,7 @@ TEST_F(DorisFSDirectoryTest, FSIndexOutputV2FlushBufferError) {
     Status s = _fs->create_file(file_path, &writer);
     EXPECT_TRUE(s.ok());
 
-    auto* output = _directory->createOutputV2(writer.get());
+    auto output = _directory->createOutputV2(writer.get());
 
     // Write small chunks to fill the buffer and trigger flush
     // BufferedIndexOutput buffer size is 1024 bytes
@@ -573,7 +573,6 @@ TEST_F(DorisFSDirectoryTest, FSIndexOutputV2FlushBufferError) {
     } catch (...) {
         // Ignore close errors in cleanup
     }
-    delete output;
 }
 
 // Test 38: FSIndexOutputV2 flushBuffer with null writer

conf/lsan_suppr.conf

@@ -20,9 +20,3 @@ leak:brpc
 leak:libjvm
 leak:libzip
 leak:*_dl_map_object_deps*
-
-# Known leaks about index
-leak:lucene::index::IndexReader::open
-leak:lucene::index::IndexWriter::IndexWriter
-leak:doris::segment_v2::IndexFileReader::open
-leak:doris::segment_v2::IndexFileWriter::close