[improvement](drop tablet) impr gc shutdown tablet lock (apache#26151) (

apache#27478)

be/src/olap/tablet_manager.cpp

@@ -638,12 +638,24 @@ TabletSharedPtr TabletManager::_get_tablet_unlocked(TTabletId tablet_id, bool in
  TabletSharedPtr tablet;
  tablet = _get_tablet_unlocked(tablet_id);
  if (tablet == nullptr && include_deleted) {
- std::shared_lock rdlock(_shutdown_tablets_lock);
- for (auto& deleted_tablet : _shutdown_tablets) {
- CHECK(deleted_tablet != nullptr) << "deleted tablet is nullptr";
- if (deleted_tablet->tablet_id() == tablet_id) {
- tablet = deleted_tablet;
- break;
+ {
+ std::shared_lock rdlock(_shutdown_tablets_lock);
+ for (auto& deleted_tablet : _shutdown_tablets) {
+ CHECK(deleted_tablet != nullptr) << "deleted tablet is nullptr";
+ if (deleted_tablet->tablet_id() == tablet_id) {
+ tablet = deleted_tablet;
+ break;
+ }
+ }
+ }
+ if (tablet == nullptr) {
+ std::shared_lock rdlock(_shutdown_deleting_tablets_lock);
+ for (auto& deleted_tablet : _shutdown_deleting_tablets) {
+ CHECK(deleted_tablet != nullptr) << "deleted tablet is nullptr";
+ if (deleted_tablet->tablet_id() == tablet_id) {
+ tablet = deleted_tablet;
+ break;
+ }
  }
  }
  }
@@ -1065,9 +1077,17 @@ Status TabletManager::start_trash_sweep() {
  clean_num = 0;
  // should get write lock here, because it will remove tablet from shut_down_tablets
  // and get tablet will access shut_down_tablets
- std::lock_guard<std::shared_mutex> wrlock(_shutdown_tablets_lock);
- auto it = _shutdown_tablets.begin();
- while (it != _shutdown_tablets.end()) {
+ {
+ std::lock_guard<std::shared_mutex> wrlock1(_shutdown_tablets_lock);
+ std::lock_guard<std::shared_mutex> wrlock2(_shutdown_deleting_tablets_lock);
+ for (const auto& tablet : _shutdown_tablets) {
+ _shutdown_deleting_tablets.push_back(tablet);
+ }
+ _shutdown_tablets.clear();
+ }
+ std::lock_guard<std::shared_mutex> wrlock(_shutdown_deleting_tablets_lock);
+ auto it = _shutdown_deleting_tablets.begin();
+ while (it != _shutdown_deleting_tablets.end()) {
  // check if the meta has the tablet info and its state is shutdown
  if (it->use_count() > 1) {
  // it means current tablet is referenced by other thread
@@ -1086,7 +1106,7 @@ Status TabletManager::start_trash_sweep() {
  << " old tablet_uid=" << (*it)->tablet_uid()
  << " cur tablet_uid=" << tablet_meta->tablet_uid();
  // remove it from list
- it = _shutdown_tablets.erase(it);
+ it = _shutdown_deleting_tablets.erase(it);
  continue;
  }
  // move data to trash
@@ -1115,7 +1135,7 @@ Status TabletManager::start_trash_sweep() {
  << "tablet_id=" << (*it)->tablet_id()
  << ", schema_hash=" << (*it)->schema_hash()
  << ", tablet_path=" << tablet_path;
- it = _shutdown_tablets.erase(it);
+ it = _shutdown_deleting_tablets.erase(it);
  ++clean_num;
  } else {
  // if could not find tablet info in meta store, then check if dir existed
@@ -1135,7 +1155,7 @@ Status TabletManager::start_trash_sweep() {
  << "tablet_id=" << (*it)->tablet_id()
  << ", schema_hash=" << (*it)->schema_hash()
  << ", tablet_path=" << tablet_path;
- it = _shutdown_tablets.erase(it);
+ it = _shutdown_deleting_tablets.erase(it);
  }
  }
 

be/src/olap/tablet_manager.h

@@ -242,6 +242,10 @@ class TabletManager {
  std::map<int64_t, std::set<TabletInfo>> _partition_tablet_map;
  std::vector<TabletSharedPtr> _shutdown_tablets;
 
+ // gc thread will move _shutdown_tablets to _shutdown_deleting_tablets
+ std::shared_mutex _shutdown_deleting_tablets_lock;
+ std::list<TabletSharedPtr> _shutdown_deleting_tablets;
+
  std::mutex _tablet_stat_cache_mutex;
  std::shared_ptr<std::vector<TTabletStat>> _tablet_stat_list_cache =
  std::make_shared<std::vector<TTabletStat>>();