diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index b77d5d6b324a32..e9956008f98179 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -91,7 +91,11 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
         _field_name = std::wstring(field_name.begin(), field_name.end());
     }
 
-    ~InvertedIndexColumnWriterImpl() override = default;
+    ~InvertedIndexColumnWriterImpl() override {
+        if (_index_writer != nullptr) {
+            close_on_error();
+        }
+    }
 
     Status init() override {
         try {
@@ -112,6 +116,7 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
     void close() {
         if (_index_writer) {
             _index_writer->close();
+            _index_writer.reset();
         }
     }