diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 545484a004e30e..616cb788a7d416 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -143,6 +143,9 @@ message(STATUS "build fs benchmark tool: ${BUILD_FS_BENCHMARK}") option(BUILD_TASK_EXECUTOR_SIMULATOR "ON for building task executor simulator or OFF for not" OFF) message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}") +option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF) +message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}") + set(CMAKE_SKIP_RPATH TRUE) set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) diff --git a/be/src/http/action/file_cache_action.cpp b/be/src/http/action/file_cache_action.cpp index 740bac46edf2a7..dbd8dcc3b5e1d9 100644 --- a/be/src/http/action/file_cache_action.cpp +++ b/be/src/http/action/file_cache_action.cpp @@ -54,6 +54,7 @@ constexpr static std::string_view CAPACITY = "capacity"; constexpr static std::string_view RELEASE = "release"; constexpr static std::string_view BASE_PATH = "base_path"; constexpr static std::string_view RELEASED_ELEMENTS = "released_elements"; +constexpr static std::string_view DUMP = "dump"; constexpr static std::string_view VALUE = "value"; Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metrics) { @@ -127,6 +128,8 @@ Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metri *json_metrics = json.ToString(); } } + } else if (operation == DUMP) { + io::FileCacheFactory::instance()->dump_all_caches(); } else { st = Status::InternalError("invalid operation: {}", operation); } diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt index f80415e0caedd3..84f40761f1ab21 100644 --- a/be/src/io/CMakeLists.txt +++ b/be/src/io/CMakeLists.txt @@ -70,3 +70,28 @@ if (${BUILD_FS_BENCHMARK} STREQUAL "ON") ) endif() + +if (${BUILD_FILE_CACHE_LRU_TOOL} STREQUAL "ON") + add_executable(file_cache_lru_tool + cache/file_cache_lru_tool.cpp + ) + + pch_reuse(file_cache_lru_tool) + + # This permits libraries loaded by dlopen to link to the symbols in the program. + set_target_properties(file_cache_lru_tool PROPERTIES ENABLE_EXPORTS 1) + + target_link_libraries(file_cache_lru_tool + ${DORIS_LINK_LIBS} + ) + + install(DIRECTORY DESTINATION ${OUTPUT_DIR}/lib/) + install(TARGETS file_cache_lru_tool DESTINATION ${OUTPUT_DIR}/lib/) + + add_custom_command(TARGET file_cache_lru_tool POST_BUILD + COMMAND ${CMAKE_OBJCOPY} --only-keep-debug $ $.dbg + COMMAND ${CMAKE_STRIP} --strip-debug --strip-unneeded $ + COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=$.dbg $ + ) + +endif() diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index 10b1b6863a4a59..2efbf6340b0ac1 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -2316,6 +2316,18 @@ void BlockFileCache::run_background_lru_log_replay() { } } +void BlockFileCache::dump_lru_queues(bool force) { + std::unique_lock dump_lock(_dump_lru_queues_mtx); + if (config::file_cache_background_lru_dump_tail_record_num > 0 && + !ExecEnv::GetInstance()->get_is_upgrading()) { + _lru_dumper->dump_queue("disposable", force); + _lru_dumper->dump_queue("normal", force); + _lru_dumper->dump_queue("index", force); + _lru_dumper->dump_queue("ttl", force); + _lru_dumper->set_first_dump_done(); + } +} + void BlockFileCache::run_background_lru_dump() { Thread::set_self_name("run_background_lru_dump"); while (!_close) { @@ -2327,14 +2339,7 @@ void BlockFileCache::run_background_lru_dump() { break; } } - - if (config::file_cache_background_lru_dump_tail_record_num > 0 && - !ExecEnv::GetInstance()->get_is_upgrading()) { - _lru_dumper->dump_queue("disposable"); - _lru_dumper->dump_queue("normal"); - _lru_dumper->dump_queue("index"); - _lru_dumper->dump_queue("ttl"); - } + dump_lru_queues(false); } } diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index d152e7403c0310..a85a36b5520802 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -211,6 +211,8 @@ class BlockFileCache { std::string dump_structure(const UInt128Wrapper& hash); std::string dump_single_cache_type(const UInt128Wrapper& hash, size_t offset); + void dump_lru_queues(bool force); + [[nodiscard]] size_t get_used_cache_size(FileCacheType type) const; [[nodiscard]] size_t get_file_blocks_num(FileCacheType type) const; @@ -556,7 +558,7 @@ class BlockFileCache { // so join this async load thread first std::unique_ptr _storage; std::shared_ptr _lru_dump_latency_us; - + std::mutex _dump_lru_queues_mtx; moodycamel::ConcurrentQueue _need_update_lru_blocks; }; diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index c9631e35dc4c79..598baa8e857e38 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -192,6 +192,12 @@ std::string FileCacheFactory::clear_file_caches(bool sync) { return ss.str(); } +void FileCacheFactory::dump_all_caches() { + for (const auto& cache : _caches) { + cache->dump_lru_queues(true); + } +} + std::vector FileCacheFactory::get_base_paths() { std::vector paths; for (const auto& pair : _path_to_cache) { diff --git a/be/src/io/cache/block_file_cache_factory.h b/be/src/io/cache/block_file_cache_factory.h index 4366afc72eb082..8b9f5ae3ccbf07 100644 --- a/be/src/io/cache/block_file_cache_factory.h +++ b/be/src/io/cache/block_file_cache_factory.h @@ -78,6 +78,11 @@ class FileCacheFactory { */ std::string clear_file_caches(bool sync); + /** + * dump lru queue info for all file cache instances + */ + void dump_all_caches(); + std::vector get_base_paths(); /** diff --git a/be/src/io/cache/cache_lru_dumper.cpp b/be/src/io/cache/cache_lru_dumper.cpp index 97d82905332449..64bbed44c1ce4c 100644 --- a/be/src/io/cache/cache_lru_dumper.cpp +++ b/be/src/io/cache/cache_lru_dumper.cpp @@ -230,10 +230,55 @@ Status CacheLRUDumper::finalize_dump(std::ofstream& out, size_t entry_num, out.close(); + if (_is_first_dump) [[unlikely]] { + // we back up two dumps (one for last before be restart, one for first after be restart) + // for later debug the restore process + try { + if (std::filesystem::exists(final_filename)) { + std::string backup_filename = final_filename + "_" + _start_time + "_last"; + std::rename(final_filename.c_str(), backup_filename.c_str()); + } + std::string timestamped_filename = final_filename + "_" + _start_time; + std::filesystem::copy_file(tmp_filename, timestamped_filename); + + std::filesystem::path dir = std::filesystem::path(final_filename).parent_path(); + std::string prefix = std::filesystem::path(final_filename).filename().string(); + uint64_t total_size = 0; + std::vector> files; + for (const auto& entry : std::filesystem::directory_iterator(dir)) { + if (entry.path().filename().string().find(prefix) == 0) { + total_size += entry.file_size(); + files.emplace_back(entry.path(), entry.last_write_time()); + } + } + if (total_size > 5ULL * 1024 * 1024 * 1024) { + // delete oldest two files + std::sort(files.begin(), files.end(), + [](const auto& a, const auto& b) { return a.second < b.second; }); + if (!files.empty()) { + auto remove_file = [](const std::filesystem::path& file_path) { + std::error_code ec; + bool removed = std::filesystem::remove(file_path, ec); + LOG(INFO) << "Remove " << (removed ? "succeeded" : "failed") + << " for file: " << file_path + << (ec ? ", error: " + ec.message() : ""); + return removed; + }; + + remove_file(files[0].first); + if (files.size() > 1) { + remove_file(files[1].first); + } + } + } + } catch (const std::filesystem::filesystem_error& e) { + LOG(WARNING) << "failed to handle first dump case: " << e.what(); + } + } + // Rename tmp to formal file try { std::rename(tmp_filename.c_str(), final_filename.c_str()); - std::remove(tmp_filename.c_str()); file_size = std::filesystem::file_size(final_filename); } catch (const std::filesystem::filesystem_error& e) { LOG(WARNING) << "failed to rename " << tmp_filename << " to " << final_filename @@ -247,10 +292,10 @@ Status CacheLRUDumper::finalize_dump(std::ofstream& out, size_t entry_num, return Status::OK(); } -void CacheLRUDumper::dump_queue(const std::string& queue_name) { +void CacheLRUDumper::dump_queue(const std::string& queue_name, bool force) { FileCacheType type = string_to_cache_type(queue_name); - if (_recorder->get_lru_queue_update_cnt_from_last_dump(type) > - config::file_cache_background_lru_dump_update_cnt_threshold) { + if (force || _recorder->get_lru_queue_update_cnt_from_last_dump(type) > + config::file_cache_background_lru_dump_update_cnt_threshold) { LRUQueue& queue = _recorder->get_shadow_queue(type); do_dump_queue(queue, queue_name); _recorder->reset_lru_queue_update_cnt_from_last_dump(type); diff --git a/be/src/io/cache/cache_lru_dumper.h b/be/src/io/cache/cache_lru_dumper.h index 801ed577de29b6..d9addff614c685 100644 --- a/be/src/io/cache/cache_lru_dumper.h +++ b/be/src/io/cache/cache_lru_dumper.h @@ -17,8 +17,10 @@ #pragma once +#include #include #include +#include #include #include #include @@ -38,11 +40,19 @@ class LRUQueueRecorder; class CacheLRUDumper { public: CacheLRUDumper(BlockFileCache* mgr, LRUQueueRecorder* recorder) - : _mgr(mgr), _recorder(recorder) {}; - void dump_queue(const std::string& queue_name); + : _mgr(mgr), _recorder(recorder) { + auto now = std::chrono::system_clock::now(); + auto in_time_t = std::chrono::system_clock::to_time_t(now); + std::stringstream ss; + ss << std::put_time(std::localtime(&in_time_t), "%Y%m%d%H%M%S"); + _start_time = ss.str(); + }; + + void dump_queue(const std::string& queue_name, bool force); void restore_queue(LRUQueue& queue, const std::string& queue_name, std::lock_guard& cache_lock); void remove_lru_dump_files(); + void set_first_dump_done() { _is_first_dump = false; } private: void do_dump_queue(LRUQueue& queue, const std::string& queue_name); @@ -79,5 +89,8 @@ class CacheLRUDumper { BlockFileCache* _mgr; LRUQueueRecorder* _recorder; + + std::string _start_time; + bool _is_first_dump = true; }; } // namespace doris::io \ No newline at end of file diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index 839d798a7c5274..261c227c2e5941 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -226,6 +226,9 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* for (auto& block : holder.file_blocks) { switch (block->state()) { case FileBlock::State::EMPTY: + VLOG_DEBUG << fmt::format("Block EMPTY path={} hash={}:{}:{} offset={} cache_path={}", + path().native(), _cache_hash.to_string(), _cache_hash.high(), + _cache_hash.low(), block->offset(), block->get_cache_file()); block->get_or_set_downloader(); if (block->is_downloader()) { empty_blocks.push_back(block); @@ -234,6 +237,10 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* stats.hit_cache = false; break; case FileBlock::State::SKIP_CACHE: + VLOG_DEBUG << fmt::format( + "Block SKIP_CACHE path={} hash={}:{}:{} offset={} cache_path={}", + path().native(), _cache_hash.to_string(), _cache_hash.high(), _cache_hash.low(), + block->offset(), block->get_cache_file()); empty_blocks.push_back(block); stats.hit_cache = false; stats.skip_cache = true; diff --git a/be/src/io/cache/file_cache_lru_tool.cpp b/be/src/io/cache/file_cache_lru_tool.cpp new file mode 100644 index 00000000000000..a6e133c7a56e35 --- /dev/null +++ b/be/src/io/cache/file_cache_lru_tool.cpp @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "common/status.h" +#include "gen_cpp/file_cache.pb.h" +#include "io/cache/cache_lru_dumper.h" +#include "io/cache/file_cache_common.h" +#include "io/cache/lru_queue_recorder.h" +#include "util/coding.h" +#include "util/crc32c.h" + +using namespace doris; + +DEFINE_string(filename, "", "dump file name"); + +std::string get_usage(const std::string& progname) { + std::stringstream ss; + ss << progname << " is the Doris BE file cache lru tool for examing dumped content.\n"; + + ss << "Usage:\n"; + ss << progname << " --filename [filename]\n"; + ss << "\nExample:\n"; + ss << progname << " --filename ./lru_dump_ttl.tail\n"; + return ss.str(); +} + +Status check_ifstream_status(std::ifstream& in, std::string& filename) { + if (!in.good()) { + std::ios::iostate state = in.rdstate(); + std::stringstream err_msg; + if (state & std::ios::eofbit) { + err_msg << "End of file reached."; + } + if (state & std::ios::failbit) { + err_msg << "Input/output operation failed, err_code: " << strerror(errno); + } + if (state & std::ios::badbit) { + err_msg << "Serious I/O error occurred, err_code: " << strerror(errno); + } + in.close(); + std::string warn_msg = std::string( + fmt::format("dump lru reading failed, file={}, {}", filename, err_msg.str())); + std::cerr << warn_msg << std::endl; + return Status::InternalError(warn_msg); + } + + return Status::OK(); +} + +struct Footer { + size_t meta_offset; + uint32_t checksum; + uint8_t version; + char magic[3]; + + std::string serialize_as_string() const; + bool deserialize_from_string(const std::string& data) { + DCHECK(data.size() == sizeof(Footer)); + + const char* ptr = data.data(); + + // Deserialize meta_offset (convert from little-endian) + uint64_t meta_offset_le; + std::memcpy(&meta_offset_le, ptr, sizeof(meta_offset_le)); + meta_offset = decode_fixed64_le(reinterpret_cast(&meta_offset_le)); + ptr += sizeof(meta_offset_le); + + // Deserialize checksum (convert from little-endian) + uint32_t checksum_le; + std::memcpy(&checksum_le, ptr, sizeof(checksum_le)); + checksum = decode_fixed32_le(reinterpret_cast(&checksum_le)); + ptr += sizeof(checksum_le); + + version = *((uint8_t*)ptr); + ptr += sizeof(version); + + // Deserialize magic + std::memcpy(magic, ptr, sizeof(magic)); + + return true; + } +} __attribute__((packed)); + +Status parse_dump_footer(std::ifstream& in, std::string& filename, size_t& entry_num, + doris::io::cache::LRUDumpMetaPb& parse_meta, + doris::io::cache::LRUDumpEntryGroupPb& current_parse_group) { + size_t file_size = std::filesystem::file_size(filename); + + // Read footer + Footer footer; + size_t footer_size = sizeof(footer); + if (file_size < footer_size) { + std::string warn_msg = std::string(fmt::format( + "LRU dump file too small to contain footer, file={}, skip restore", filename)); + std::cerr << warn_msg << std::endl; + return Status::InternalError(warn_msg); + } + + in.seekg(-footer_size, std::ios::end); + std::string footer_str(footer_size, '\0'); + in.read(&footer_str[0], footer_size); + RETURN_IF_ERROR(check_ifstream_status(in, filename)); + + if (!footer.deserialize_from_string(footer_str)) { + std::string warn_msg = std::string( + fmt::format("Failed to deserialize footer, file={}, skip restore", filename)); + std::cerr << warn_msg << std::endl; + return Status::InternalError(warn_msg); + } + + // Validate footer + if (footer.version != 1 || std::string(footer.magic, 3) != "DOR") { + std::string warn_msg = std::string(fmt::format( + "LRU dump file invalid footer format, file={}, skip restore", filename)); + std::cerr << warn_msg << std::endl; + return Status::InternalError(warn_msg); + } + + // Read meta + in.seekg(footer.meta_offset, std::ios::beg); + size_t meta_size = file_size - footer.meta_offset - footer_size; + if (meta_size <= 0) { + std::string warn_msg = std::string( + fmt::format("LRU dump file invalid meta size, file={}, skip restore", filename)); + std::cerr << warn_msg << std::endl; + return Status::InternalError(warn_msg); + } + std::string meta_serialized(meta_size, '\0'); + in.read(&meta_serialized[0], meta_serialized.size()); + RETURN_IF_ERROR(check_ifstream_status(in, filename)); + parse_meta.Clear(); + current_parse_group.Clear(); + if (!parse_meta.ParseFromString(meta_serialized)) { + std::string warn_msg = std::string( + fmt::format("LRU dump file meta parse failed, file={}, skip restore", filename)); + std::cerr << warn_msg << std::endl; + return Status::InternalError(warn_msg); + } + std::cout << "parse meta: " << parse_meta.DebugString() << std::endl; + + entry_num = parse_meta.entry_num(); + return Status::OK(); +} + +Status parse_one_lru_entry(std::ifstream& in, std::string& filename, io::UInt128Wrapper& hash, + size_t& offset, size_t& size, + doris::io::cache::LRUDumpMetaPb& parse_meta, + doris::io::cache::LRUDumpEntryGroupPb& current_parse_group) { + // Read next group if current is empty + if (current_parse_group.entries_size() == 0) { + if (parse_meta.group_offset_size_size() == 0) { + return Status::EndOfFile("No more entries"); + } + + auto group_info = parse_meta.group_offset_size(0); + in.seekg(group_info.offset(), std::ios::beg); + std::string group_serialized(group_info.size(), '\0'); + in.read(&group_serialized[0], group_serialized.size()); + RETURN_IF_ERROR(check_ifstream_status(in, filename)); + uint32_t checksum = crc32c::Value(group_serialized.data(), group_serialized.size()); + if (checksum != group_info.checksum()) { + std::string warn_msg = + fmt::format("restore lru failed as checksum not match, file={}", filename); + std::cerr << warn_msg << std::endl; + return Status::InternalError(warn_msg); + } + if (!current_parse_group.ParseFromString(group_serialized)) { + std::string warn_msg = + fmt::format("restore lru failed to parse group, file={}", filename); + std::cerr << warn_msg << std::endl; + return Status::InternalError(warn_msg); + } + + // Remove processed group info + parse_meta.mutable_group_offset_size()->erase(parse_meta.group_offset_size().begin()); + } + + // Get next entry from current group + std::cout << "After deserialization: " << current_parse_group.DebugString() << std::endl; + auto entry = current_parse_group.entries(0); + hash = io::UInt128Wrapper((static_cast(entry.hash().high()) << 64) | + entry.hash().low()); + offset = entry.offset(); + size = entry.size(); + + std::cout << hash.to_string() << " " << offset << " " << size << std::endl; + + // Remove processed entry + current_parse_group.mutable_entries()->erase(current_parse_group.entries().begin()); + return Status::OK(); +} + +int main(int argc, char** argv) { + std::string usage = get_usage(argv[0]); + gflags::SetUsageMessage(usage); + google::ParseCommandLineFlags(&argc, &argv, true); + + std::ifstream in(FLAGS_filename, std::ios::binary); + size_t entry_num; + doris::io::cache::LRUDumpMetaPb parse_meta; + doris::io::cache::LRUDumpEntryGroupPb current_parse_group; + auto s = parse_dump_footer(in, FLAGS_filename, entry_num, parse_meta, current_parse_group); + + in.seekg(0, std::ios::beg); + io::UInt128Wrapper hash; + size_t offset, size; + for (int i = 0; i < entry_num; ++i) { + EXIT_IF_ERROR(parse_one_lru_entry(in, FLAGS_filename, hash, offset, size, parse_meta, + current_parse_group)); + } + + return 0; +} diff --git a/build.sh b/build.sh index 98e9b67188999e..a55471bca726d5 100755 --- a/build.sh +++ b/build.sh @@ -582,11 +582,16 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then BUILD_TASK_EXECUTOR_SIMULATOR=OFF fi + if [[ -z "${BUILD_FILE_CACHE_LRU_TOOL}" ]]; then + BUILD_FILE_CACHE_LRU_TOOL=OFF + fi + echo "-- Make program: ${MAKE_PROGRAM}" echo "-- Use ccache: ${CMAKE_USE_CCACHE}" echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}" echo "-- Build fs benchmark tool: ${BUILD_FS_BENCHMARK}" echo "-- Build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}" + echo "-- Build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}" mkdir -p "${CMAKE_BUILD_DIR}" cd "${CMAKE_BUILD_DIR}" @@ -600,6 +605,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DBUILD_BENCHMARK="${BUILD_BENCHMARK}" \ -DBUILD_FS_BENCHMARK="${BUILD_FS_BENCHMARK}" \ -DBUILD_TASK_EXECUTOR_SIMULATOR="${BUILD_TASK_EXECUTOR_SIMULATOR}" \ + -DBUILD_FILE_CACHE_LRU_TOOL="${BUILD_FILE_CACHE_LRU_TOOL}" \ ${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \ -DUSE_LIBCPP="${USE_LIBCPP}" \ -DBUILD_META_TOOL="${BUILD_META_TOOL}" \ diff --git a/regression-test/suites/demo_p0/test_lru_persist.groovy b/regression-test/suites/demo_p0/test_lru_persist.groovy index 249faadeedad95..d039c74c5c59b7 100644 --- a/regression-test/suites/demo_p0/test_lru_persist.groovy +++ b/regression-test/suites/demo_p0/test_lru_persist.groovy @@ -46,7 +46,6 @@ import org.apache.doris.regression.suite.ClusterOptions suite('test_lru_persist', 'docker') { def options = new ClusterOptions() - options.feNum = 1 options.beNum = 1 options.msNum = 1 @@ -90,5 +89,39 @@ suite('test_lru_persist', 'docker') { logger.info("normalAfter: ${normalAfter}") assert normalBefore == normalAfter + + // remove dump file + def rm_dump_ret = "rm -rf ${cachePath}/lru_dump_normal.tail".execute().text.trim() + cluster.startBackends(1) + sleep(5000) + def show_backend_ret = sql '''show backends''' + try { + logger.info("Backend details: ${show_backend_ret.toString()}") + if(show_backend_ret.size() > 0 && show_backend_ret[0].size() > 0) { + logger.info("alive: ${show_backend_ret[0][9].toString()}") + } + assert show_backend_ret[0][9].toString() == "true" + } catch(Exception e) { + logger.error("Failed to log backend info: ${e.message}") + } + + sql '''select count(*) from tb1''' + + sleep(10000) + cluster.stopBackends(1) + + def rm_data_ret = new File(cachePath).eachFile { file -> + if (!file.name.startsWith("lru_") && file.name != "version" && file.name != "." && file.name != "..") { + if (file.isDirectory()) { + file.deleteDir() + } else { + file.delete() + } + } + } + cluster.startBackends(1) + sleep(5000) + + sql '''select count(*) from tb1''' } -} +} \ No newline at end of file