diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 3ec5d22166477f9..289a4ec0d62750d 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -943,7 +943,7 @@ void TabletSchema::remove_index(int64_t index_id) {
     _indexes = std::move(indexes);
 }
 
-void TabletSchema::clear_columns() {
+void TabletSchema::clear_columns(bool release_handler) {
     _field_path_to_index.clear();
     _field_name_to_index.clear();
     _field_id_to_index.clear();
@@ -952,7 +952,9 @@ void TabletSchema::clear_columns() {
     _num_null_columns = 0;
     _num_key_columns = 0;
     _cols.clear();
-    clear_column_cache_handlers();
+    if (release_handler) {
+        clear_column_cache_handlers();
+    }
 }
 
 void TabletSchema::clear_column_cache_handlers() {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index c813d6f0ef8722b..dff226101f04fe3 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -426,7 +426,7 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
     void update_indexes_from_thrift(const std::vector<doris::TOlapTableIndex>& indexes);
     // If schema version is not set, it should be -1
     int32_t schema_version() const { return _schema_version; }
-    void clear_columns();
+    void clear_columns(bool release_handler = true);
     vectorized::Block create_block(
             const std::vector<uint32_t>& return_columns,
             const std::unordered_set<uint32_t>* tablet_columns_need_convert_null = nullptr) const;
diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp
index 78c1d0779a3f376..d9bbe449c5af55f 100644
--- a/be/src/vec/common/schema_util.cpp
+++ b/be/src/vec/common/schema_util.cpp
@@ -416,8 +416,9 @@ Status get_least_common_schema(const std::vector<TabletSchemaSPtr>& schemas,
     // duplicated paths following the update_least_common_schema process.
     auto build_schema_without_extracted_columns = [&](const TabletSchemaSPtr& base_schema) {
         output_schema = std::make_shared<TabletSchema>(*base_schema);
-        // Merge columns from other schemas
-        output_schema->clear_columns();
+        // Merge columns from other schemas, not release original column handlers
+        // otherwise lead to double free
+        output_schema->clear_columns(false);
         // Get all columns without extracted columns and collect variant col unique id
         for (const TabletColumnPtr& col : base_schema->columns()) {
             if (col->is_variant_type()) {