From 8ac412857f66d68a3a6b7e572b1cbb073c7ac3af Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 20 Jun 2025 11:04:35 +0800 Subject: [PATCH 1/3] [fix](inverted index) fix error when writing empty index file (#51984) Related PR: #51393 Problem Summary: This PR ensures that copying an empty inverted index segment no longer throws an exception. --- .../segment_v2/inverted_index_file_writer.cpp | 4 ++ .../inverted_index_file_writer_test.cpp | 59 ++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index 30cf7a73cf3e20..4595d0e27c7b4d 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -206,6 +206,10 @@ void InvertedIndexFileWriter::copyFile(const char* fileName, lucene::store::Dire err.set(CL_ERR_IO, "debug point: copyFile_openInput_error"); }); if (!open) { + if (err.number() == CL_ERR_EmptyIndexSegment) { + LOG(WARNING) << "InvertedIndexFileWriter::copyFile: " << fileName << " is empty"; + return; + } throw err; } diff --git a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp index 6022ad0a8579b2..d98e43b0b169a3 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp @@ -601,5 +601,60 @@ TEST_F(InvertedIndexFileWriterTest, WriteV2ExceptionHandlingTest) { ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR); } -} // namespace segment_v2 -} // namespace doris +TEST_F(InvertedIndexFileWriterTest, CopyFileEmptyFileTest) { + // This test uses the existing MockDorisFSDirectoryOpenInput class + auto mock_dir = std::make_shared(); + std::string local_fs_index_path = InvertedIndexDescriptor::get_temporary_index_path( + ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir().native(), _rowset_id, + _seg_id, 1, "suffix1"); + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(local_fs_index_path).ok()); + EXPECT_TRUE(io::global_local_filesystem()->create_directory(local_fs_index_path).ok()); + mock_dir->init(_fs, local_fs_index_path.c_str()); + + // Create test files + std::vector files = {"_0.fnm", "_0.frq", "_0.tii"}; + for (const auto& file : files) { + auto out_file = + std::unique_ptr(mock_dir->createOutput(file.c_str())); + out_file->writeString("test content"); + out_file->close(); + } + + InvertedIndexFileWriter writer(_fs, _index_path_prefix, _rowset_id, _seg_id, + InvertedIndexStorageFormatPB::V2); + + // Setup mock to simulate CL_ERR_EmptyIndexSegment error for _0.frq file + EXPECT_CALL(*mock_dir, + openInput(::testing::StrEq("_0.frq"), ::testing::_, ::testing::_, ::testing::_)) + .WillOnce(::testing::Invoke([&](const char* name, lucene::store::IndexInput*& ret, + CLuceneError& err_ref, int32_t bufferSize) { + err_ref.set(CL_ERR_EmptyIndexSegment, + std::string("Empty index segment for file: ").append(name).c_str()); + return false; + })); + + // Test copyFile with the mock directory + uint8_t buffer[16384]; + + // Create a temporary output stream to test copyFile + auto* ram_dir = new lucene::store::RAMDirectory(); + auto* output = ram_dir->createOutput("test_output"); + + // Before the fix, this would throw an exception + // After the fix, this should log a warning and return gracefully + EXPECT_NO_THROW({ writer.copyFile("_0.frq", mock_dir.get(), output, buffer, sizeof(buffer)); }); + + // The output should remain valid (not corrupted by the exception) + EXPECT_NE(output, nullptr); + + // Clean up + output->close(); + _CLLDELETE(output); + ram_dir->close(); + _CLLDELETE(ram_dir); + + // Clean up the test directory + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(local_fs_index_path).ok()); +} + +} // namespace doris::segment_v2 From 76e2e6430a72f245c618d60bdbbabf836ee91f3a Mon Sep 17 00:00:00 2001 From: hezhiqiang Date: Wed, 2 Jul 2025 21:35:09 +0800 Subject: [PATCH 2/3] fix compile --- .../olap/rowset/segment_v2/inverted_index_file_writer_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp index d98e43b0b169a3..9987ab6e040621 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp @@ -658,3 +658,4 @@ TEST_F(InvertedIndexFileWriterTest, CopyFileEmptyFileTest) { } } // namespace doris::segment_v2 +} \ No newline at end of file From e2a22350595eea84ee09578f3301a6322cb55b0f Mon Sep 17 00:00:00 2001 From: hezhiqiang Date: Wed, 2 Jul 2025 21:36:05 +0800 Subject: [PATCH 3/3] fmt --- .../rowset/segment_v2/inverted_index_file_writer_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp index 9987ab6e040621..2aead76daee6fc 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp @@ -657,5 +657,5 @@ TEST_F(InvertedIndexFileWriterTest, CopyFileEmptyFileTest) { EXPECT_TRUE(io::global_local_filesystem()->delete_directory(local_fs_index_path).ok()); } -} // namespace doris::segment_v2 -} \ No newline at end of file +} // namespace segment_v2 +} // namespace doris \ No newline at end of file