diff --git a/dbms/src/Storages/Page/CMakeLists.txt b/dbms/src/Storages/Page/CMakeLists.txt index 627ee63e2c5..48e2dcbb675 100644 --- a/dbms/src/Storages/Page/CMakeLists.txt +++ b/dbms/src/Storages/Page/CMakeLists.txt @@ -1 +1,11 @@ add_subdirectory(V2) + +# PageStorage Stress test +if (ENABLE_V3_PAGESTORAGE) + add_headers_and_sources(page_stress_testing stress) + add_headers_and_sources(page_stress_testing stress/workload) + add_executable(page_stress_testing EXCLUDE_FROM_ALL ${page_stress_testing_sources}) + target_link_libraries(page_stress_testing dbms page_storage_v3) + target_include_directories(page_stress_testing PRIVATE stress) + target_compile_options(page_stress_testing PRIVATE -Wno-format -lc++) # turn off printf format check +endif() \ No newline at end of file diff --git a/dbms/src/Storages/Page/V2/tests/CMakeLists.txt b/dbms/src/Storages/Page/V2/tests/CMakeLists.txt index 7eec3332349..4e900d1299d 100644 --- a/dbms/src/Storages/Page/V2/tests/CMakeLists.txt +++ b/dbms/src/Storages/Page/V2/tests/CMakeLists.txt @@ -15,14 +15,6 @@ target_compile_options(gtests_page_storage PRIVATE -Wno-unknown-pragmas) target_compile_definitions(gtests_page_storage PRIVATE DBMS_PUBLIC_GTEST) add_check(gtests_page_storage) -# non googletest -add_headers_and_sources(page_stress_testing stress) -add_headers_and_sources(page_stress_testing stress/workload) -add_executable(page_stress_testing EXCLUDE_FROM_ALL ${page_stress_testing_sources}) -target_link_libraries(page_stress_testing page_storage_v2) -target_include_directories(page_stress_testing PRIVATE stress) -target_compile_options(page_stress_testing PRIVATE -Wno-format -lc++) # turn off printf format check - # FIXME: mock disk full by FailPoint, and move this test into a unittest case. add_executable(test_page_storage_write_disk_full test_page_storage_write_disk_full.cpp) target_link_libraries(test_page_storage_write_disk_full dbms) diff --git a/dbms/src/Storages/Page/V2/tests/stress/PSBackground.cpp b/dbms/src/Storages/Page/stress/PSBackground.cpp similarity index 100% rename from dbms/src/Storages/Page/V2/tests/stress/PSBackground.cpp rename to dbms/src/Storages/Page/stress/PSBackground.cpp diff --git a/dbms/src/Storages/Page/V2/tests/stress/PSBackground.h b/dbms/src/Storages/Page/stress/PSBackground.h similarity index 100% rename from dbms/src/Storages/Page/V2/tests/stress/PSBackground.h rename to dbms/src/Storages/Page/stress/PSBackground.h diff --git a/dbms/src/Storages/Page/V2/tests/stress/PSRunnable.cpp b/dbms/src/Storages/Page/stress/PSRunnable.cpp similarity index 97% rename from dbms/src/Storages/Page/V2/tests/stress/PSRunnable.cpp rename to dbms/src/Storages/Page/stress/PSRunnable.cpp index 72d68fd8f6c..5a3507e9074 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/PSRunnable.cpp +++ b/dbms/src/Storages/Page/stress/PSRunnable.cpp @@ -66,13 +66,13 @@ DB::ReadBufferPtr PSWriter::genRandomData(const DB::PageId pageId, DB::MemHolder holder = DB::createMemHolder(buff, [&](char * p) { free(p); }); - return std::make_shared(buff, buff_sz); + return std::make_shared(const_cast(buff), buff_sz); } void PSWriter::updatedRandomData() { size_t memory_size = approx_page_mb * DB::MB * 2; - if (unlikely(memory != nullptr)) + if (memory == nullptr) { memory = static_cast(malloc(memory_size)); if (memory == nullptr) @@ -131,7 +131,7 @@ void PSCommonWriter::updatedRandomData() : batch_buffer_size); size_t memory_size = single_buff_size * batch_buffer_nums; - if (likely(memory == nullptr)) + if (memory == nullptr) { memory = static_cast(malloc(memory_size)); if (memory == nullptr) @@ -170,7 +170,6 @@ bool PSCommonWriter::runImpl() bytes_used += buffptr->buffer().size(); } - writing_page[index] = page_id; ps->write(std::move(wb)); return (batch_buffer_limit == 0 || bytes_used < batch_buffer_limit); } @@ -302,7 +301,10 @@ DB::PageId PSWindowWriter::genRandomPageId() auto random = std::round(distribution(gen)); // Move this "random" near the pageid_boundary, If "random" is still negative, then make it positive random = std::abs(random + pageid_boundary); - return static_cast(random > pageid_boundary ? pageid_boundary++ : random); + + auto page_id = static_cast(random > pageid_boundary ? pageid_boundary++ : random); + writing_page[index] = page_id; + return page_id; } void PSWindowReader::setWindowSize(size_t window_size_) diff --git a/dbms/src/Storages/Page/V2/tests/stress/PSRunnable.h b/dbms/src/Storages/Page/stress/PSRunnable.h similarity index 99% rename from dbms/src/Storages/Page/V2/tests/stress/PSRunnable.h rename to dbms/src/Storages/Page/stress/PSRunnable.h index 5cd45469dc9..f5901d0844d 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/PSRunnable.h +++ b/dbms/src/Storages/Page/stress/PSRunnable.h @@ -254,5 +254,5 @@ class PSSnapshotReader : public PSReader protected: size_t snapshots_hold_num; size_t snapshot_get_interval_ms = 0; - std::list snapshots; + std::list snapshots; }; \ No newline at end of file diff --git a/dbms/src/Storages/Page/V2/tests/stress/PSStressEnv.cpp b/dbms/src/Storages/Page/stress/PSStressEnv.cpp similarity index 90% rename from dbms/src/Storages/Page/V2/tests/stress/PSStressEnv.cpp rename to dbms/src/Storages/Page/stress/PSStressEnv.cpp index bcd1045523c..b3de67ac0e5 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/PSStressEnv.cpp +++ b/dbms/src/Storages/Page/stress/PSStressEnv.cpp @@ -43,7 +43,8 @@ StressEnv StressEnv::parse(int argc, char ** argv) ("failpoints,F", value>(), "failpoint(s) to enable") // ("status_interval,S", value()->default_value(1), "Status statistics interval. 0 means no statistics") // ("situation_mask,M", value()->default_value(0), "Run special tests sequentially, example -M 2") // - ("verify", value()->default_value(true), "Run special tests sequentially with verify."); // + ("verify", value()->default_value(true), "Run special tests sequentially with verify.") // + ("running_ps_version,V", value()->default_value(3), "Select a version of PageStorage. 2 or 3 can used"); po::variables_map options; po::store(po::parse_command_line(argc, argv, desc), options); @@ -68,6 +69,14 @@ StressEnv StressEnv::parse(int argc, char ** argv) opt.status_interval = options["status_interval"].as(); opt.situation_mask = options["situation_mask"].as(); opt.verify = options["verify"].as(); + opt.running_ps_version = options["running_ps_version"].as(); + + if (opt.running_ps_version != 2 && opt.running_ps_version != 3) + { + std::cerr << "Invalid running_ps_version, this arg should be 2 or 3." << std::endl; + std::cerr << desc << std::endl; + exit(0); + } if (options.count("paths")) opt.paths = options["paths"].as>(); @@ -93,6 +102,8 @@ void StressEnv::setup() #ifdef FIU_ENABLE fiu_init(0); #endif + + for (const auto & fp : failpoints) { DB::FailPointHelper::enableFailPoint(fp); diff --git a/dbms/src/Storages/Page/V2/tests/stress/PSStressEnv.h b/dbms/src/Storages/Page/stress/PSStressEnv.h similarity index 91% rename from dbms/src/Storages/Page/V2/tests/stress/PSStressEnv.h rename to dbms/src/Storages/Page/stress/PSStressEnv.h index d9612d7f3bf..a4e74ed146b 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/PSStressEnv.h +++ b/dbms/src/Storages/Page/stress/PSStressEnv.h @@ -1,18 +1,17 @@ #pragma once #include -#include +#include #include #include -using namespace DB::PS::V2; namespace Poco { class Logger; } -using PSPtr = std::shared_ptr; +using PSPtr = std::shared_ptr; enum StressEnvStat { @@ -70,6 +69,7 @@ struct StressEnv size_t status_interval = 1; size_t situation_mask = 0; bool verify = true; + size_t running_ps_version = 3; std::vector paths; std::vector failpoints; @@ -81,7 +81,8 @@ struct StressEnv "num_writers: {}, num_readers: {}, init_pages: {}, clean_before_run: {}" ", timeout_s: {}, read_delay_ms: {}, num_writer_slots: {}" ", avg_page_size_mb: {}, paths: [{}], failpoints: [{}]" - ", status_interval: {}, situation_mask: {}, verify: {}." + ", status_interval: {}, situation_mask: {}, verify: {}" + ", running_pagestorage_version : {}." "}}", num_writers, num_readers, @@ -95,7 +96,8 @@ struct StressEnv fmt::join(failpoints.begin(), failpoints.end(), ","), status_interval, situation_mask, - verify + verify, + running_ps_version // ); } diff --git a/dbms/src/Storages/Page/V2/tests/stress/PSWorkload.cpp b/dbms/src/Storages/Page/stress/PSWorkload.cpp similarity index 86% rename from dbms/src/Storages/Page/V2/tests/stress/PSWorkload.cpp rename to dbms/src/Storages/Page/stress/PSWorkload.cpp index bbbb128eaa8..5cdd27c1711 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/PSWorkload.cpp +++ b/dbms/src/Storages/Page/stress/PSWorkload.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include void StressWorkload::onDumpResult() @@ -47,7 +49,7 @@ void StressWorkload::onDumpResult() } } -void StressWorkload::initPageStorage(PageStorage::Config & config, String path_prefix) +void StressWorkload::initPageStorage(DB::PageStorage::Config & config, String path_prefix) { DB::FileProviderPtr file_provider = std::make_shared(std::make_shared(false), false); @@ -67,8 +69,22 @@ void StressWorkload::initPageStorage(PageStorage::Config & config, String path_p delegator = std::make_shared(options.paths[0] + "/" + path_prefix); } - ps = std::make_shared("stress_test", delegator, config, file_provider); - ps->restore(); + if (options.running_ps_version == 2) + { + ps = std::make_shared("stress_test", delegator, config, file_provider); + ps->restore(); + } + else if (options.running_ps_version == 3) + { + ps = std::make_shared("stress_test", delegator, config, file_provider); + // FIXME : we need restore V3 + } + else + { + throw DB::Exception(fmt::format("Invalid PageStorage version {}", + options.running_ps_version)); + } + { size_t num_of_pages = 0; ps->traverse([&num_of_pages](const DB::Page & page) { diff --git a/dbms/src/Storages/Page/V2/tests/stress/PSWorkload.h b/dbms/src/Storages/Page/stress/PSWorkload.h similarity index 97% rename from dbms/src/Storages/Page/V2/tests/stress/PSWorkload.h rename to dbms/src/Storages/Page/stress/PSWorkload.h index 3dba87c25b7..e24bac54f44 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/PSWorkload.h +++ b/dbms/src/Storages/Page/stress/PSWorkload.h @@ -6,11 +6,10 @@ #include #include #include -#include +#include #include #define NORMAL_WORKLOAD 0 -using namespace DB::PS::V2; template class StressWorkloadFunc { @@ -47,7 +46,7 @@ class StressWorkload virtual void onDumpResult(); protected: - void initPageStorage(PageStorage::Config & config, String path_prefix = ""); + void initPageStorage(DB::PageStorage::Config & config, String path_prefix = ""); void startBackgroundTimer(); diff --git a/dbms/src/Storages/Page/V2/tests/stress/stress_page_storage.cpp b/dbms/src/Storages/Page/stress/stress_page_storage.cpp similarity index 100% rename from dbms/src/Storages/Page/V2/tests/stress/stress_page_storage.cpp rename to dbms/src/Storages/Page/stress/stress_page_storage.cpp diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/HeavyMemoryCostInGC.cpp b/dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp similarity index 97% rename from dbms/src/Storages/Page/V2/tests/stress/workload/HeavyMemoryCostInGC.cpp rename to dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp index 38c6a25a08e..fc446fd0f6c 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/HeavyMemoryCostInGC.cpp +++ b/dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp @@ -33,7 +33,7 @@ class HeavyMemoryCostInGC { stop_watch.start(); - PageStorage::Config config; + DB::PageStorage::Config config; initPageStorage(config, name()); metrics_dumper = std::make_shared(1); diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/HeavyRead.cpp b/dbms/src/Storages/Page/stress/workload/HeavyRead.cpp similarity index 97% rename from dbms/src/Storages/Page/V2/tests/stress/workload/HeavyRead.cpp rename to dbms/src/Storages/Page/stress/workload/HeavyRead.cpp index 4460af28d87..14f45745b7a 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/HeavyRead.cpp +++ b/dbms/src/Storages/Page/stress/workload/HeavyRead.cpp @@ -32,7 +32,7 @@ class HeavyRead : public StressWorkload void run() override { - PageStorage::Config config; + DB::PageStorage::Config config; initPageStorage(config, name()); PSWriter::fillAllPages(ps); diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/HeavySkewWriteRead.cpp b/dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp similarity index 98% rename from dbms/src/Storages/Page/V2/tests/stress/workload/HeavySkewWriteRead.cpp rename to dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp index bfcfa7a811e..a98b97a87c2 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/HeavySkewWriteRead.cpp +++ b/dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp @@ -32,7 +32,7 @@ class HeavySkewWriteRead : public StressWorkload void run() override { pool.addCapacity(1 + options.num_writers + options.num_readers); - PageStorage::Config config; + DB::PageStorage::Config config; initPageStorage(config, name()); metrics_dumper = std::make_shared(1); diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/HeavyWrite.cpp b/dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp similarity index 97% rename from dbms/src/Storages/Page/V2/tests/stress/workload/HeavyWrite.cpp rename to dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp index 38ab4713c46..c0f453436d3 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/HeavyWrite.cpp +++ b/dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp @@ -31,7 +31,7 @@ class HeavyWrite : public StressWorkload void run() override { - PageStorage::Config config; + DB::PageStorage::Config config; initPageStorage(config, name()); metrics_dumper = std::make_shared(1); diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/HighValidBigFileGC.cpp b/dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp similarity index 97% rename from dbms/src/Storages/Page/V2/tests/stress/workload/HighValidBigFileGC.cpp rename to dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp index 68fea677203..6827b731b0e 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/HighValidBigFileGC.cpp +++ b/dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp @@ -42,7 +42,7 @@ class HighValidBigFileGCWorkload { stop_watch.start(); - PageStorage::Config config; + DB::PageStorage::Config config; config.file_max_size = 8ULL * DB::GB; config.file_roll_size = 8ULL * DB::GB; initPageStorage(config, name()); @@ -63,7 +63,7 @@ class HighValidBigFileGCWorkload // Generate normal data in the same Pagefile { - PageStorage::Config config; + DB::PageStorage::Config config; config.file_max_size = DB::PAGE_FILE_MAX_SIZE; config.file_roll_size = DB::PAGE_FILE_ROLL_SIZE; initPageStorage(config, name()); diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/HoldSnapshotsLongTime.cpp b/dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp similarity index 98% rename from dbms/src/Storages/Page/V2/tests/stress/workload/HoldSnapshotsLongTime.cpp rename to dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp index 80c5b7d6c78..325092ead83 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/HoldSnapshotsLongTime.cpp +++ b/dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp @@ -33,7 +33,7 @@ class HoldSnapshotsLongTime : public StressWorkload void run() override { pool.addCapacity(1 + options.num_writers + options.num_readers); - PageStorage::Config config; + DB::PageStorage::Config config; initPageStorage(config, name()); metrics_dumper = std::make_shared(1); diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/Normal.cpp b/dbms/src/Storages/Page/stress/workload/Normal.cpp similarity index 97% rename from dbms/src/Storages/Page/V2/tests/stress/workload/Normal.cpp rename to dbms/src/Storages/Page/stress/workload/Normal.cpp index e4f75d8e35e..7d60d8355c2 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/Normal.cpp +++ b/dbms/src/Storages/Page/stress/workload/Normal.cpp @@ -28,7 +28,7 @@ class NormalWorkload { pool.addCapacity(1 + options.num_writers + options.num_readers); - PageStorage::Config config; + DB::PageStorage::Config config; config.num_write_slots = options.num_writer_slots; initPageStorage(config); diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/PageStorageInMemoryCapacity.cpp b/dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp similarity index 99% rename from dbms/src/Storages/Page/V2/tests/stress/workload/PageStorageInMemoryCapacity.cpp rename to dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp index cf3c9c15a36..896fe007b0d 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/PageStorageInMemoryCapacity.cpp +++ b/dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp @@ -119,7 +119,7 @@ class PageStorageInMemoryCapacity : public StressWorkload std::tie(total_mem, begin_virtual_size, begin_resident_size) = getCurrentMemory(); pool.addCapacity(1 + options.num_writers + options.num_readers); - PageStorage::Config config; + DB::PageStorage::Config config; initPageStorage(config, name()); metrics_dumper = std::make_shared(1); diff --git a/dbms/src/Storages/Page/V2/tests/stress/workload/ThousandsOfOffset.cpp b/dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp similarity index 99% rename from dbms/src/Storages/Page/V2/tests/stress/workload/ThousandsOfOffset.cpp rename to dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp index c1d7787b7b0..e30d71958be 100644 --- a/dbms/src/Storages/Page/V2/tests/stress/workload/ThousandsOfOffset.cpp +++ b/dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp @@ -54,7 +54,7 @@ class ThousandsOfOffset : public StressWorkload void run() override { pool.addCapacity(1 + options.num_writers + options.num_readers); - PageStorage::Config config; + DB::PageStorage::Config config; initPageStorage(config, name()); metrics_dumper = std::make_shared(1);