From d4a75510fcf99176f5c0ee1723a027e3e79a96ba Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 9 Apr 2025 17:27:04 +0300 Subject: [PATCH 01/31] Added date_from&date_to timestamp uniform distribution for 'ydb workload log' --- ydb/library/workload/log/log.cpp | 20 ++++++++++++++++++-- ydb/library/workload/log/log.h | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 7339adce9acd..17a827396234 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -1,4 +1,5 @@ #include "log.h" +#include #include #include #include @@ -252,6 +253,11 @@ class TRandomLogGenerator { return result.str(); } + TInstant UniformInstant(ui64 from, ui64 to) const { + TMersenne rnd(Seed()); + return TInstant::FromValue(rnd.Uniform(from, to)); + } + TInstant RandomInstant() const { auto result = TInstant::Now() - TDuration::Seconds(Params.TimestampSubtract); i64 millisecondsDiff = 60 * 1000 * NormalRandom(0., Params.TimestampStandardDeviationMinutes); @@ -279,7 +285,7 @@ class TRandomLogGenerator { for (size_t row = 0; row < count; ++row) { result.emplace_back(); result.back().LogId = CreateGuidAsString().c_str(); - result.back().Ts = RandomInstant(); + result.back().Ts = Params.TimestampDateFrom.has_value() ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant(); result.back().Level = RandomNumber(10); result.back().ServiceName = RandomWord(false); result.back().Component = RandomWord(true); @@ -419,9 +425,19 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT opts.AddLongOption("rows", "Number of rows to upsert") .DefaultValue(RowsCnt).StoreResult(&RowsCnt); opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") - .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes); + // .DefaultValue(TimestampStandardDeviationMinutes) + .Optional() + .StoreResult(&TimestampStandardDeviationMinutes); + + opts.AddLongOption("date-from", "TODO").Optional().StoreResult(&TimestampDateFrom); + opts.AddLongOption("date-to", "TODO").Optional().StoreResult(&TimestampDateTo); + + opts.MutuallyExclusive("date-from", "timestamp_deviation"); + opts.MutuallyExclusive("date-to", "timestamp_deviation"); + opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") .DefaultValue(0).StoreResult(&TimestampSubtract); + opts.AddLongOption("null-percent", "Percent of nulls in generated data") .DefaultValue(NullPercent).StoreResult(&NullPercent); break; diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 7145a0e04791..aa2feb9b6988 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -24,6 +24,8 @@ class TLogWorkloadParams : public TWorkloadParams { ui64 IntColumnsCnt = 0; ui64 KeyColumnsCnt = 0; ui64 TimestampStandardDeviationMinutes = 0; + std::optional TimestampDateFrom{std::nullopt}; + std::optional TimestampDateTo{std::nullopt}; ui64 TimestampTtlMinutes = 0; ui64 TimestampSubtract = 0; ui64 RowsCnt = 1; From d059fcd6496671b09c2c32f39f79c5c7d7bd923e Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 9 Apr 2025 20:01:50 +0300 Subject: [PATCH 02/31] enthropy + description --- ydb/library/workload/log/log.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 17a827396234..5d6b89465f86 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -1,4 +1,5 @@ #include "log.h" +#include #include #include #include @@ -429,15 +430,17 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT .Optional() .StoreResult(&TimestampStandardDeviationMinutes); - opts.AddLongOption("date-from", "TODO").Optional().StoreResult(&TimestampDateFrom); - opts.AddLongOption("date-to", "TODO").Optional().StoreResult(&TimestampDateTo); - - opts.MutuallyExclusive("date-from", "timestamp_deviation"); - opts.MutuallyExclusive("date-to", "timestamp_deviation"); + opts.AddLongOption("date-from", "Left boundary of the interval to generate " + "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " + "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") + .Optional().StoreResult(&TimestampDateFrom); + opts.AddLongOption("date-to", "Right boundary of the interval to generate " + "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " + "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") + .Optional().StoreResult(&TimestampDateTo); opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") .DefaultValue(0).StoreResult(&TimestampSubtract); - opts.AddLongOption("null-percent", "Percent of nulls in generated data") .DefaultValue(NullPercent).StoreResult(&NullPercent); break; From a7bf9f261ac53c4eb7a0681f57b2d2df9724772c Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Thu, 10 Apr 2025 15:38:00 +0300 Subject: [PATCH 03/31] mutually_exclusive options --- ydb/library/workload/log/log.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 5d6b89465f86..2a4ed890ea26 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -438,6 +438,9 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") .Optional().StoreResult(&TimestampDateTo); + + opts.MutuallyExclusive("timestamp_deviation", "date-from"); + opts.MutuallyExclusive("timestamp_deviation", "date-to"); opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") .DefaultValue(0).StoreResult(&TimestampSubtract); From a85d2e933b434e9231f6681735f3d45ab2eee2b9 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Thu, 24 Apr 2025 13:57:07 +0300 Subject: [PATCH 04/31] added workload log run parse param validation for PK mode --- .../workload/abstract/workload_query_generator.h | 2 ++ ydb/library/workload/log/log.cpp | 16 ++++++++++++++++ ydb/library/workload/log/log.h | 1 + ydb/public/lib/ydb_cli/commands/ydb_workload.cpp | 4 ++++ ydb/public/lib/ydb_cli/commands/ydb_workload.h | 1 + 5 files changed, 24 insertions(+) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index 71d4a5ad4f5a..e959551fd89f 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -172,6 +173,7 @@ class TWorkloadParams { virtual ~TWorkloadParams() = default; virtual void ConfigureOpts(NLastGetopt::TOpts& /*opts*/, const ECommandType /*commandType*/, int /*workloadType*/) { }; + virtual void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& /*config*/) {}; virtual THolder CreateGenerator() const = 0; virtual TWorkloadDataInitializer::TList CreateDataInitializers() const { return {}; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 2a4ed890ea26..d473f7f3b0d9 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -429,6 +429,7 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT // .DefaultValue(TimestampStandardDeviationMinutes) .Optional() .StoreResult(&TimestampStandardDeviationMinutes); + // TODO: maybe it shoudn't be optional opts.AddLongOption("date-from", "Left boundary of the interval to generate " "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " @@ -472,6 +473,21 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT } } +void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) { + // TODO: should I check if it's a specific command type: run ? + auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation"); + auto date_from_passed = config.ParseResult->Has("date-from"); + auto date_to_passed = config.ParseResult->Has("date-to"); + + if (timestamp_dev_passed && (date_from_passed || date_to_passed)) { + throw yexception() << "timestamp_deviation and date_from, date_to are mutually exclusive and shouldn't be passed at once"; + } + + if ((date_from_passed && !date_to_passed) || (!date_from_passed && date_to_passed)) { + throw yexception() << "The `date_from` and `date_to` parameters must be provided together to specify the interval for uniform PK generation"; + } +} + THolder TLogWorkloadParams::CreateGenerator() const { return MakeHolder(this); } diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index aa2feb9b6988..46d77f646ce0 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -14,6 +14,7 @@ class TLogWorkloadParams : public TWorkloadParams { }; void ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) override; + void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) override; THolder CreateGenerator() const override; TString GetWorkloadName() const override; ui64 MinPartitions = 40; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index a5ce8a927349..e5f9c8af59b8 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -374,6 +374,10 @@ void TWorkloadCommandBase::Config(TConfig& config) { Params.ConfigureOpts(config.Opts->GetOpts(), CommandType, Type); } +void TWorkloadCommandBase::Parse(TConfig& config) { + Params.Parse(config); +} + int TWorkloadCommandBase::Run(TConfig& config) { Driver = MakeHolder(CreateDriver(config)); if (!DryRun) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.h b/ydb/public/lib/ydb_cli/commands/ydb_workload.h index 909e6a72a5fe..64e345a9b94a 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.h @@ -89,6 +89,7 @@ class TWorkloadCommandBase: public TYdbCommand { const TString& description = TString(), int type = 0); virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override final; protected: From 71dfa29bbe059deddacff03b58efae1e8eaa7e7c Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Thu, 24 Apr 2025 17:47:43 +0300 Subject: [PATCH 05/31] fix `macro redifined` --- ydb/library/workload/tpch/driver.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ydb/library/workload/tpch/driver.h b/ydb/library/workload/tpch/driver.h index 52b3a23ae475..c079a14be5b5 100644 --- a/ydb/library/workload/tpch/driver.h +++ b/ydb/library/workload/tpch/driver.h @@ -6,7 +6,13 @@ extern "C" { #include #include +#ifdef DT_CHR +#undef DT_CHR +#endif #include +#ifdef DT_CHR +#undef DT_CHR +#endif #include void InitTpchGen(DSS_HUGE scale); From 39fefbfb4f04e60078d6ef642daa51f4b6ab6d5a Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Fri, 25 Apr 2025 12:29:13 +0300 Subject: [PATCH 06/31] Commented library mutually exclusive settings --- ydb/library/workload/log/log.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index d473f7f3b0d9..7ae2212196b6 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -440,8 +440,8 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") .Optional().StoreResult(&TimestampDateTo); - opts.MutuallyExclusive("timestamp_deviation", "date-from"); - opts.MutuallyExclusive("timestamp_deviation", "date-to"); + // opts.MutuallyExclusive("timestamp_deviation", "date-from"); + // opts.MutuallyExclusive("timestamp_deviation", "date-to"); opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") .DefaultValue(0).StoreResult(&TimestampSubtract); @@ -480,7 +480,7 @@ void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& co auto date_to_passed = config.ParseResult->Has("date-to"); if (timestamp_dev_passed && (date_from_passed || date_to_passed)) { - throw yexception() << "timestamp_deviation and date_from, date_to are mutually exclusive and shouldn't be passed at once"; + throw yexception() << "The `timestamp_deviation` and `date_from`, `date_to` are mutually exclusive and shouldn't be passed at once"; } if ((date_from_passed && !date_to_passed) || (!date_from_passed && date_to_passed)) { From 5dddd6a31113c50d0b336f747f3168a59b20e96f Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Fri, 25 Apr 2025 13:22:38 +0300 Subject: [PATCH 07/31] Moved Parse method to appropriate WorkloadRunCommand class & added interval validation --- ydb/library/workload/log/log.cpp | 10 ++++++++++ ydb/public/lib/ydb_cli/commands/ydb_workload.cpp | 8 ++++---- ydb/public/lib/ydb_cli/commands/ydb_workload.h | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 7ae2212196b6..88c3b75e5886 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -286,6 +286,7 @@ class TRandomLogGenerator { for (size_t row = 0; row < count; ++row) { result.emplace_back(); result.back().LogId = CreateGuidAsString().c_str(); + // TODO: check if it's correct to get interval params here result.back().Ts = Params.TimestampDateFrom.has_value() ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant(); result.back().Level = RandomNumber(10); result.back().ServiceName = RandomWord(false); @@ -486,6 +487,15 @@ void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& co if ((date_from_passed && !date_to_passed) || (!date_from_passed && date_to_passed)) { throw yexception() << "The `date_from` and `date_to` parameters must be provided together to specify the interval for uniform PK generation"; } + + if (date_from_passed && date_to_passed) { + auto date_from_val = config.ParseResult->Get("date_from"); + auto date_to_val = config.ParseResult->Get("date_to"); + + if (date_from_val >= date_to_val) { + throw yexception() << "Invalid interval [`date_from`, `date_to`)"; + } + } } THolder TLogWorkloadParams::CreateGenerator() const { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index e5f9c8af59b8..acfd2be727d1 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -359,6 +359,10 @@ void TWorkloadCommandRun::Config(TConfig& config) { Params.ConfigureOpts(config.Opts->GetOpts(), NYdbWorkload::TWorkloadParams::ECommandType::Run, Type); } +void TWorkloadCommandRun::Parse(TConfig& config) { + Params.Parse(config); +} + TWorkloadCommandBase::TWorkloadCommandBase(const TString& name, NYdbWorkload::TWorkloadParams& params, const NYdbWorkload::TWorkloadParams::ECommandType commandType, const TString& description, int type) : TYdbCommand(name, std::initializer_list(), description) , CommandType(commandType) @@ -374,10 +378,6 @@ void TWorkloadCommandBase::Config(TConfig& config) { Params.ConfigureOpts(config.Opts->GetOpts(), CommandType, Type); } -void TWorkloadCommandBase::Parse(TConfig& config) { - Params.Parse(config); -} - int TWorkloadCommandBase::Run(TConfig& config) { Driver = MakeHolder(CreateDriver(config)); if (!DryRun) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.h b/ydb/public/lib/ydb_cli/commands/ydb_workload.h index 64e345a9b94a..ece29031e546 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.h @@ -73,6 +73,7 @@ class TWorkloadCommandRun : public TWorkloadCommand { public: TWorkloadCommandRun(NYdbWorkload::TWorkloadParams& params, const NYdbWorkload::IWorkloadQueryGenerator::TWorkloadType& workload); virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; private: @@ -89,7 +90,6 @@ class TWorkloadCommandBase: public TYdbCommand { const TString& description = TString(), int type = 0); virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override final; protected: From 377e77713ce3aeef7ab9c08bd7690bbcb6a2a8fa Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Mon, 28 Apr 2025 15:01:44 +0300 Subject: [PATCH 08/31] Some changes --- ydb/library/workload/log/log.cpp | 12 ++++++++++-- ydb/public/lib/ydb_cli/common/command.cpp | 6 +++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 88c3b75e5886..85885b502964 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -480,12 +480,20 @@ void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& co auto date_from_passed = config.ParseResult->Has("date-from"); auto date_to_passed = config.ParseResult->Has("date-to"); + Cerr << "timestamp_dev_passed: " << timestamp_dev_passed << Endl; + Cerr << "date_from_passed: " << date_from_passed << Endl; + Cerr << "date_to_passed: " << date_to_passed << Endl; + + if (!timestamp_dev_passed && (!date_from_passed && !date_to_passed)) { + throw yexception() << "One of parameter should be provided"; + } + if (timestamp_dev_passed && (date_from_passed || date_to_passed)) { - throw yexception() << "The `timestamp_deviation` and `date_from`, `date_to` are mutually exclusive and shouldn't be passed at once"; + throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once"; } if ((date_from_passed && !date_to_passed) || (!date_from_passed && date_to_passed)) { - throw yexception() << "The `date_from` and `date_to` parameters must be provided together to specify the interval for uniform PK generation"; + throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation"; } if (date_from_passed && date_to_passed) { diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 93053be7bf44..03309f1d6d21 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -263,7 +263,11 @@ void TClientCommand::Prepare(TConfig& config) { SetCustomUsage(config); SaveParseResult(config); config.ParseResult = ParseResult.get(); - Parse(config); + try { + Parse(config); + } catch (const std::exception& e) { + Cerr << "Error occured during Parse() stage: " << e.what(); + } } void TClientCommand::ExtractParams(TConfig& config) { From 21ef45c26dc82454edba10d2e3be6e28d87cec23 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Mon, 5 May 2025 11:24:51 +0300 Subject: [PATCH 09/31] debug logs --- ydb/public/lib/ydb_cli/common/client_command_options.cpp | 1 + ydb/public/lib/ydb_cli/common/command.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/ydb/public/lib/ydb_cli/common/client_command_options.cpp b/ydb/public/lib/ydb_cli/common/client_command_options.cpp index d6932f4d801b..8014e76b9a3b 100644 --- a/ydb/public/lib/ydb_cli/common/client_command_options.cpp +++ b/ydb/public/lib/ydb_cli/common/client_command_options.cpp @@ -422,6 +422,7 @@ TOptionsParseResult::TOptionsParseResult(const TClientCommandOptions* options, i , ParseFromCommandLineResult(&options->GetOpts(), argc, argv, throwOnParseError) { for (const auto& clientOption : ClientOptions->ClientOpts) { + Cerr << clientOption->GetOpt().GetName() << "\n"; if (const auto* optResult = ParseFromCommandLineResult.FindOptParseResult(&clientOption->GetOpt())) { Opts.emplace_back(clientOption, optResult); if (dynamic_cast(clientOption.Get())) { diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 03309f1d6d21..77be1eec6295 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -236,6 +236,7 @@ int TClientCommand::Process(TConfig& config) { } void TClientCommand::SaveParseResult(TConfig& config) { + Cerr << "ParseResult called\n"; ParseResult = std::make_shared(config.Opts, config.ArgC, (const char**)config.ArgV, config.ThrowOnOptsParseError); // Parse options from env and apply default parameters. From 3548f77e51ac273729d6a7e50f6b9f5c08176f12 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Tue, 6 May 2025 18:11:32 +0300 Subject: [PATCH 10/31] modifying ParseResult->Has method to check if name is presented in ClientOpts --- ydb/library/workload/log/log.cpp | 4 ++-- ydb/public/lib/ydb_cli/commands/ydb_workload.cpp | 8 ++++++++ ydb/public/lib/ydb_cli/common/client_command_options.cpp | 8 +++++++- ydb/public/lib/ydb_cli/common/command.cpp | 5 +++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 85885b502964..48f42f687863 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -484,8 +484,8 @@ void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& co Cerr << "date_from_passed: " << date_from_passed << Endl; Cerr << "date_to_passed: " << date_to_passed << Endl; - if (!timestamp_dev_passed && (!date_from_passed && !date_to_passed)) { - throw yexception() << "One of parameter should be provided"; + if (!timestamp_dev_passed && (!date_from_passed || !date_to_passed)) { + throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; } if (timestamp_dev_passed && (date_from_passed || date_to_passed)) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index acfd2be727d1..9f105b54a94b 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -360,6 +360,14 @@ void TWorkloadCommandRun::Config(TConfig& config) { } void TWorkloadCommandRun::Parse(TConfig& config) { + TWorkloadCommand::Parse(config); + // auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation"); + // auto date_from_passed = config.ParseResult->Has("date-from"); + // auto date_to_passed = config.ParseResult->Has("date-to"); + + // Cerr << "____timestamp_dev_passed: " << timestamp_dev_passed << Endl; + // Cerr << "____date_from_passed: " << date_from_passed << Endl; + // Cerr << "____date_to_passed: " << date_to_passed << Endl; Params.Parse(config); } diff --git a/ydb/public/lib/ydb_cli/common/client_command_options.cpp b/ydb/public/lib/ydb_cli/common/client_command_options.cpp index 8014e76b9a3b..b507284c9692 100644 --- a/ydb/public/lib/ydb_cli/common/client_command_options.cpp +++ b/ydb/public/lib/ydb_cli/common/client_command_options.cpp @@ -422,7 +422,7 @@ TOptionsParseResult::TOptionsParseResult(const TClientCommandOptions* options, i , ParseFromCommandLineResult(&options->GetOpts(), argc, argv, throwOnParseError) { for (const auto& clientOption : ClientOptions->ClientOpts) { - Cerr << clientOption->GetOpt().GetName() << "\n"; + // Cerr << clientOption->GetOpt().GetName() << "\n"; if (const auto* optResult = ParseFromCommandLineResult.FindOptParseResult(&clientOption->GetOpt())) { Opts.emplace_back(clientOption, optResult); if (dynamic_cast(clientOption.Get())) { @@ -476,6 +476,9 @@ bool TOptionsParseResult::Has(const TString& name, bool includeDefault) const { if (!includeDefault && result && result->ValueSource == EOptionValueSource::DefaultValue) { result = nullptr; } + if (result == nullptr) { + return ClientOptions->GetOpts().FindLongOption(name) != nullptr; + } return result != nullptr; } @@ -484,6 +487,9 @@ bool TOptionsParseResult::Has(char name, bool includeDefault) const { if (!includeDefault && result && result->ValueSource == EOptionValueSource::DefaultValue) { result = nullptr; } + if (result == nullptr) { + return ClientOptions->GetOpts().FindCharOption(name) != nullptr; + } return result != nullptr; } diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 77be1eec6295..6f18953ce31e 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -238,6 +238,11 @@ int TClientCommand::Process(TConfig& config) { void TClientCommand::SaveParseResult(TConfig& config) { Cerr << "ParseResult called\n"; ParseResult = std::make_shared(config.Opts, config.ArgC, (const char**)config.ArgV, config.ThrowOnOptsParseError); + Cerr << "Opts: ["; + for (auto x: config.Opts->GetOpts().GetOpts()) { + Cerr << x->GetName() << ", "; + } + Cerr << "]\n"; // Parse options from env and apply default parameters. // Parsing from profiles is only supported at high level commands and occure in ExtractParams() stage. From b8f456daaa8c58d77d866331c7674ef6f3131b07 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Tue, 6 May 2025 20:14:15 +0300 Subject: [PATCH 11/31] Changed Has to get provided args --- ydb/public/lib/ydb_cli/commands/ydb_workload.cpp | 12 ++++++------ .../lib/ydb_cli/common/client_command_options.cpp | 7 +++++-- ydb/public/lib/ydb_cli/common/command.cpp | 10 +++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index 9f105b54a94b..36af5c792def 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -361,13 +361,13 @@ void TWorkloadCommandRun::Config(TConfig& config) { void TWorkloadCommandRun::Parse(TConfig& config) { TWorkloadCommand::Parse(config); - // auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation"); - // auto date_from_passed = config.ParseResult->Has("date-from"); - // auto date_to_passed = config.ParseResult->Has("date-to"); + auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation"); + auto date_from_passed = config.ParseResult->Has("date-from"); + auto date_to_passed = config.ParseResult->Has("date-to"); - // Cerr << "____timestamp_dev_passed: " << timestamp_dev_passed << Endl; - // Cerr << "____date_from_passed: " << date_from_passed << Endl; - // Cerr << "____date_to_passed: " << date_to_passed << Endl; + Cerr << "____timestamp_dev_passed: " << timestamp_dev_passed << Endl; + Cerr << "____date_from_passed: " << date_from_passed << Endl; + Cerr << "____date_to_passed: " << date_to_passed << Endl; Params.Parse(config); } diff --git a/ydb/public/lib/ydb_cli/common/client_command_options.cpp b/ydb/public/lib/ydb_cli/common/client_command_options.cpp index b507284c9692..b2bb8942b238 100644 --- a/ydb/public/lib/ydb_cli/common/client_command_options.cpp +++ b/ydb/public/lib/ydb_cli/common/client_command_options.cpp @@ -477,7 +477,7 @@ bool TOptionsParseResult::Has(const TString& name, bool includeDefault) const { result = nullptr; } if (result == nullptr) { - return ClientOptions->GetOpts().FindLongOption(name) != nullptr; + return ParseFromCommandLineResult.FindLongOptParseResult(name); } return result != nullptr; } @@ -488,7 +488,7 @@ bool TOptionsParseResult::Has(char name, bool includeDefault) const { result = nullptr; } if (result == nullptr) { - return ClientOptions->GetOpts().FindCharOption(name) != nullptr; + return ParseFromCommandLineResult.FindCharOptParseResult(name); } return result != nullptr; } @@ -496,6 +496,9 @@ bool TOptionsParseResult::Has(char name, bool includeDefault) const { const TString& TOptionsParseResult::Get(const TString& name, bool includeDefault) const { if (const TOptionParseResult* result = FindResult(name); result && (includeDefault || result->ValueSource != EOptionValueSource::DefaultValue)) { return result->Values().back(); + } + if (auto Opt = ParseFromCommandLineResult.Get(name)) { + } throw yexception() << "No \"" << name << "\" option"; } diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 6f18953ce31e..82268a327373 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -269,11 +269,11 @@ void TClientCommand::Prepare(TConfig& config) { SetCustomUsage(config); SaveParseResult(config); config.ParseResult = ParseResult.get(); - try { - Parse(config); - } catch (const std::exception& e) { - Cerr << "Error occured during Parse() stage: " << e.what(); - } + // try { + Parse(config); + // } catch (const std::exception& e) { + // Cerr << "Error occured during Parse() stage: " << e.what(); + // } } void TClientCommand::ExtractParams(TConfig& config) { From 9c30cf9224bae5bb01e9c96da37c737ea9358546 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 7 May 2025 14:36:33 +0300 Subject: [PATCH 12/31] Added validation & changed ParseFromCommandLineResult Get logic --- ydb/library/workload/log/log.cpp | 13 +++++++++---- .../lib/ydb_cli/common/client_command_options.cpp | 6 +++--- .../lib/ydb_cli/common/client_command_options.h | 2 +- ydb/public/lib/ydb_cli/common/command.cpp | 10 +++++----- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 48f42f687863..e1f579e6c282 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -497,11 +497,16 @@ void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& co } if (date_from_passed && date_to_passed) { - auto date_from_val = config.ParseResult->Get("date_from"); - auto date_to_val = config.ParseResult->Get("date_to"); + auto date_from_val = config.ParseResult->Get("date-from"); + auto date_to_val = config.ParseResult->Get("date-to"); - if (date_from_val >= date_to_val) { - throw yexception() << "Invalid interval [`date_from`, `date_to`)"; + ui64 date_from, date_to; + if (TryFromString(date_from_val, date_from) && TryFromString(date_to_val, date_to)) { + if (date_from >= date_to) { + throw yexception() << "Invalid interval [`date-from`, `date-to`)"; + } + } else { + throw yexception() << "Can't parse `date-from`, `date-to` parameters"; } } } diff --git a/ydb/public/lib/ydb_cli/common/client_command_options.cpp b/ydb/public/lib/ydb_cli/common/client_command_options.cpp index b2bb8942b238..ee10d22e38a2 100644 --- a/ydb/public/lib/ydb_cli/common/client_command_options.cpp +++ b/ydb/public/lib/ydb_cli/common/client_command_options.cpp @@ -493,12 +493,12 @@ bool TOptionsParseResult::Has(char name, bool includeDefault) const { return result != nullptr; } -const TString& TOptionsParseResult::Get(const TString& name, bool includeDefault) const { +TString TOptionsParseResult::Get(const TString& name, bool includeDefault) const { if (const TOptionParseResult* result = FindResult(name); result && (includeDefault || result->ValueSource != EOptionValueSource::DefaultValue)) { return result->Values().back(); } - if (auto Opt = ParseFromCommandLineResult.Get(name)) { - + if (auto opt = ParseFromCommandLineResult.FindLongOptParseResult(name)) { + return opt->Back(); } throw yexception() << "No \"" << name << "\" option"; } diff --git a/ydb/public/lib/ydb_cli/common/client_command_options.h b/ydb/public/lib/ydb_cli/common/client_command_options.h index 03c6d0ac77cb..8abfc4c15b5c 100644 --- a/ydb/public/lib/ydb_cli/common/client_command_options.h +++ b/ydb/public/lib/ydb_cli/common/client_command_options.h @@ -399,7 +399,7 @@ class TOptionsParseResult { bool Has(const TString& name, bool includeDefault = false) const; bool Has(char name, bool includeDefault = false) const; - const TString& Get(const TString& name, bool includeDefault = true) const; + TString Get(const TString& name, bool includeDefault = true) const; const NLastGetopt::TOptsParseResult& GetCommandLineParseResult() const { return ParseFromCommandLineResult; diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 82268a327373..68f24fab4943 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -238,11 +238,11 @@ int TClientCommand::Process(TConfig& config) { void TClientCommand::SaveParseResult(TConfig& config) { Cerr << "ParseResult called\n"; ParseResult = std::make_shared(config.Opts, config.ArgC, (const char**)config.ArgV, config.ThrowOnOptsParseError); - Cerr << "Opts: ["; - for (auto x: config.Opts->GetOpts().GetOpts()) { - Cerr << x->GetName() << ", "; - } - Cerr << "]\n"; + // Cerr << "Opts: ["; + // for (auto x: config.Opts->GetOpts().GetOpts()) { + // Cerr << x->GetName() << ", "; + // } + // Cerr << "]\n"; // Parse options from env and apply default parameters. // Parsing from profiles is only supported at high level commands and occure in ExtractParams() stage. From ae8895de36767c8a7497dc2b1e4ea7a73398d46e Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 7 May 2025 14:55:14 +0300 Subject: [PATCH 13/31] refactoring --- ydb/library/workload/log/log.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index e1f579e6c282..0ee43116f644 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -428,18 +428,17 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT .DefaultValue(RowsCnt).StoreResult(&RowsCnt); opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") // .DefaultValue(TimestampStandardDeviationMinutes) - .Optional() .StoreResult(&TimestampStandardDeviationMinutes); // TODO: maybe it shoudn't be optional opts.AddLongOption("date-from", "Left boundary of the interval to generate " "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") - .Optional().StoreResult(&TimestampDateFrom); + .StoreResult(&TimestampDateFrom); opts.AddLongOption("date-to", "Right boundary of the interval to generate " "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") - .Optional().StoreResult(&TimestampDateTo); + .StoreResult(&TimestampDateTo); // opts.MutuallyExclusive("timestamp_deviation", "date-from"); // opts.MutuallyExclusive("timestamp_deviation", "date-to"); From 3449bd82f1f67a3b53584507aab50d25766a0d55 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 7 May 2025 15:24:28 +0300 Subject: [PATCH 14/31] Refactoring --- ydb/library/workload/log/log.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 0ee43116f644..63c2315611cb 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -287,7 +287,7 @@ class TRandomLogGenerator { result.emplace_back(); result.back().LogId = CreateGuidAsString().c_str(); // TODO: check if it's correct to get interval params here - result.back().Ts = Params.TimestampDateFrom.has_value() ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant(); + result.back().Ts = Params.TimestampDateFrom.has_value() && Params.TimestampDateTo.has_value() ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant(); result.back().Level = RandomNumber(10); result.back().ServiceName = RandomWord(false); result.back().Component = RandomWord(true); @@ -427,9 +427,7 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT opts.AddLongOption("rows", "Number of rows to upsert") .DefaultValue(RowsCnt).StoreResult(&RowsCnt); opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") - // .DefaultValue(TimestampStandardDeviationMinutes) .StoreResult(&TimestampStandardDeviationMinutes); - // TODO: maybe it shoudn't be optional opts.AddLongOption("date-from", "Left boundary of the interval to generate " "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " @@ -439,9 +437,6 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") .StoreResult(&TimestampDateTo); - - // opts.MutuallyExclusive("timestamp_deviation", "date-from"); - // opts.MutuallyExclusive("timestamp_deviation", "date-to"); opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") .DefaultValue(0).StoreResult(&TimestampSubtract); From b6c131d6d871eab0f3e637b46176db4433709638 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 7 May 2025 17:39:48 +0300 Subject: [PATCH 15/31] avoided ydb_cli dependency in workload abstract lib --- .../abstract/workload_query_generator.h | 4 +- ydb/library/workload/log/log.cpp | 48 ++++++++++++++++--- ydb/library/workload/log/log.h | 3 +- .../lib/ydb_cli/commands/ydb_workload.cpp | 5 +- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index e959551fd89f..f2851259b4bd 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -173,7 +172,8 @@ class TWorkloadParams { virtual ~TWorkloadParams() = default; virtual void ConfigureOpts(NLastGetopt::TOpts& /*opts*/, const ECommandType /*commandType*/, int /*workloadType*/) { }; - virtual void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& /*config*/) {}; + // virtual void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& /*config*/) {}; + virtual void Parse(const NLastGetopt::TOptsParseResult& /*opts*/) {}; virtual THolder CreateGenerator() const = 0; virtual TWorkloadDataInitializer::TList CreateDataInitializers() const { return {}; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 63c2315611cb..bcf4ca5e10a5 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -282,6 +282,13 @@ class TRandomLogGenerator { TVector GenerateRandomRows(ui64 count) const { TVector result; result.reserve(count); + Cerr << "Debug : TimestampDeviation value: " << Params.TimestampStandardDeviationMinutes << "\n"; + if (Params.TimestampDateFrom.has_value()) { + Cerr << "Debug: DateFrom: " << *Params.TimestampDateFrom << "\n"; + } + if (Params.TimestampDateTo.has_value()) { + Cerr << "Debug: DateTo: " << *Params.TimestampDateTo << "\n"; + } for (size_t row = 0; row < count; ++row) { result.emplace_back(); @@ -468,11 +475,16 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT } } -void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) { +// void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) { +void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& opts) { // TODO: should I check if it's a specific command type: run ? - auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation"); - auto date_from_passed = config.ParseResult->Has("date-from"); - auto date_to_passed = config.ParseResult->Has("date-to"); + auto timestamp_dev_passed = opts.Has("timestamp_deviation"); + auto date_from_passed = opts.Has("date-from"); + auto date_to_passed = opts.Has("date-to"); + + // auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation"); + // auto date_from_passed = config.ParseResult->Has("date-from"); + // auto date_to_passed = config.ParseResult->Has("date-to"); Cerr << "timestamp_dev_passed: " << timestamp_dev_passed << Endl; Cerr << "date_from_passed: " << date_from_passed << Endl; @@ -491,8 +503,10 @@ void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& co } if (date_from_passed && date_to_passed) { - auto date_from_val = config.ParseResult->Get("date-from"); - auto date_to_val = config.ParseResult->Get("date-to"); + auto date_from_val = opts.Get("date-from"); + // auto date_from_val = config.ParseResult->Get("date-from"); + auto date_to_val = opts.Get("date-to"); + // auto date_to_val = config.ParseResult->Get("date-to"); ui64 date_from, date_to; if (TryFromString(date_from_val, date_from) && TryFromString(date_to_val, date_to)) { @@ -503,6 +517,28 @@ void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& co throw yexception() << "Can't parse `date-from`, `date-to` parameters"; } } + + // DEBUG + if (timestamp_dev_passed) { + auto timestamp_dev_val = opts.Get("timestamp_deviation"); + ui64 timestamp_deviation; + TryFromString(timestamp_dev_val, timestamp_deviation) && TryFromString(timestamp_dev_val, timestamp_deviation); + Cerr << "timestamp_dev val: " << timestamp_deviation << "\n"; + } + + if (date_from_passed) { + auto date_from_val = opts.Get("date-from"); + ui64 date_from; + TryFromString(date_from_val, date_from) && TryFromString(date_from_val, date_from); + Cerr << "date_from val: " << date_from << "\n"; + } + + if (date_from_passed) { + auto date_to_val = opts.Get("date-to"); + ui64 date_to; + TryFromString(date_to_val, date_to) && TryFromString(date_to_val, date_to); + Cerr << "date_to val: " << date_to << "\n"; + } } THolder TLogWorkloadParams::CreateGenerator() const { diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 46d77f646ce0..34af4890fbef 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -14,7 +14,8 @@ class TLogWorkloadParams : public TWorkloadParams { }; void ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) override; - void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) override; + // void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) override; + void Parse(const NLastGetopt::TOptsParseResult& opts) override; THolder CreateGenerator() const override; TString GetWorkloadName() const override; ui64 MinPartitions = 40; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index 36af5c792def..ff4c3dd8d267 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -368,7 +368,10 @@ void TWorkloadCommandRun::Parse(TConfig& config) { Cerr << "____timestamp_dev_passed: " << timestamp_dev_passed << Endl; Cerr << "____date_from_passed: " << date_from_passed << Endl; Cerr << "____date_to_passed: " << date_to_passed << Endl; - Params.Parse(config); + + Params.Parse(ParseResult->GetCommandLineParseResult()); + + // Params.Parse(config); } TWorkloadCommandBase::TWorkloadCommandBase(const TString& name, NYdbWorkload::TWorkloadParams& params, const NYdbWorkload::TWorkloadParams::ECommandType commandType, const TString& description, int type) From 65395f27e173913e9cdb87e1156f45426a35c62c Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 7 May 2025 17:43:29 +0300 Subject: [PATCH 16/31] Rolled back client_command_options changes, it's redundant now --- ydb/public/lib/ydb_cli/common/client_command_options.cpp | 9 --------- ydb/public/lib/ydb_cli/common/client_command_options.h | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/ydb/public/lib/ydb_cli/common/client_command_options.cpp b/ydb/public/lib/ydb_cli/common/client_command_options.cpp index ee10d22e38a2..b6a1c43a47fc 100644 --- a/ydb/public/lib/ydb_cli/common/client_command_options.cpp +++ b/ydb/public/lib/ydb_cli/common/client_command_options.cpp @@ -476,9 +476,6 @@ bool TOptionsParseResult::Has(const TString& name, bool includeDefault) const { if (!includeDefault && result && result->ValueSource == EOptionValueSource::DefaultValue) { result = nullptr; } - if (result == nullptr) { - return ParseFromCommandLineResult.FindLongOptParseResult(name); - } return result != nullptr; } @@ -487,9 +484,6 @@ bool TOptionsParseResult::Has(char name, bool includeDefault) const { if (!includeDefault && result && result->ValueSource == EOptionValueSource::DefaultValue) { result = nullptr; } - if (result == nullptr) { - return ParseFromCommandLineResult.FindCharOptParseResult(name); - } return result != nullptr; } @@ -497,9 +491,6 @@ TString TOptionsParseResult::Get(const TString& name, bool includeDefault) const if (const TOptionParseResult* result = FindResult(name); result && (includeDefault || result->ValueSource != EOptionValueSource::DefaultValue)) { return result->Values().back(); } - if (auto opt = ParseFromCommandLineResult.FindLongOptParseResult(name)) { - return opt->Back(); - } throw yexception() << "No \"" << name << "\" option"; } diff --git a/ydb/public/lib/ydb_cli/common/client_command_options.h b/ydb/public/lib/ydb_cli/common/client_command_options.h index 8abfc4c15b5c..03c6d0ac77cb 100644 --- a/ydb/public/lib/ydb_cli/common/client_command_options.h +++ b/ydb/public/lib/ydb_cli/common/client_command_options.h @@ -399,7 +399,7 @@ class TOptionsParseResult { bool Has(const TString& name, bool includeDefault = false) const; bool Has(char name, bool includeDefault = false) const; - TString Get(const TString& name, bool includeDefault = true) const; + const TString& Get(const TString& name, bool includeDefault = true) const; const NLastGetopt::TOptsParseResult& GetCommandLineParseResult() const { return ParseFromCommandLineResult; From f948b2f52d94f287f0bc8170e6f0017b62f2d210 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 7 May 2025 19:13:35 +0300 Subject: [PATCH 17/31] removed comments & debug --- .../abstract/workload_query_generator.h | 3 +- ydb/library/workload/log/log.cpp | 54 +++---------------- ydb/library/workload/log/log.h | 1 - .../lib/ydb_cli/commands/ydb_workload.cpp | 10 ---- .../ydb_cli/common/client_command_options.cpp | 3 +- ydb/public/lib/ydb_cli/common/command.cpp | 9 ---- 6 files changed, 8 insertions(+), 72 deletions(-) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index f2851259b4bd..1c9516db2a03 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -172,8 +172,7 @@ class TWorkloadParams { virtual ~TWorkloadParams() = default; virtual void ConfigureOpts(NLastGetopt::TOpts& /*opts*/, const ECommandType /*commandType*/, int /*workloadType*/) { }; - // virtual void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& /*config*/) {}; - virtual void Parse(const NLastGetopt::TOptsParseResult& /*opts*/) {}; + virtual void Parse(const NLastGetopt::TOptsParseResult& /*parseResult*/) {}; virtual THolder CreateGenerator() const = 0; virtual TWorkloadDataInitializer::TList CreateDataInitializers() const { return {}; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index bcf4ca5e10a5..c4ae6b703967 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -282,18 +282,10 @@ class TRandomLogGenerator { TVector GenerateRandomRows(ui64 count) const { TVector result; result.reserve(count); - Cerr << "Debug : TimestampDeviation value: " << Params.TimestampStandardDeviationMinutes << "\n"; - if (Params.TimestampDateFrom.has_value()) { - Cerr << "Debug: DateFrom: " << *Params.TimestampDateFrom << "\n"; - } - if (Params.TimestampDateTo.has_value()) { - Cerr << "Debug: DateTo: " << *Params.TimestampDateTo << "\n"; - } for (size_t row = 0; row < count; ++row) { result.emplace_back(); result.back().LogId = CreateGuidAsString().c_str(); - // TODO: check if it's correct to get interval params here result.back().Ts = Params.TimestampDateFrom.has_value() && Params.TimestampDateTo.has_value() ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant(); result.back().Level = RandomNumber(10); result.back().ServiceName = RandomWord(false); @@ -475,20 +467,10 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT } } -// void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) { -void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& opts) { - // TODO: should I check if it's a specific command type: run ? - auto timestamp_dev_passed = opts.Has("timestamp_deviation"); - auto date_from_passed = opts.Has("date-from"); - auto date_to_passed = opts.Has("date-to"); - - // auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation"); - // auto date_from_passed = config.ParseResult->Has("date-from"); - // auto date_to_passed = config.ParseResult->Has("date-to"); - - Cerr << "timestamp_dev_passed: " << timestamp_dev_passed << Endl; - Cerr << "date_from_passed: " << date_from_passed << Endl; - Cerr << "date_to_passed: " << date_to_passed << Endl; +void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& parseResults) { + auto timestamp_dev_passed = parseResults.Has("timestamp_deviation"); + auto date_from_passed = parseResults.Has("date-from"); + auto date_to_passed = parseResults.Has("date-to"); if (!timestamp_dev_passed && (!date_from_passed || !date_to_passed)) { throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; @@ -503,10 +485,8 @@ void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& opts) { } if (date_from_passed && date_to_passed) { - auto date_from_val = opts.Get("date-from"); - // auto date_from_val = config.ParseResult->Get("date-from"); - auto date_to_val = opts.Get("date-to"); - // auto date_to_val = config.ParseResult->Get("date-to"); + auto date_from_val = parseResults.Get("date-from"); + auto date_to_val = parseResults.Get("date-to"); ui64 date_from, date_to; if (TryFromString(date_from_val, date_from) && TryFromString(date_to_val, date_to)) { @@ -517,28 +497,6 @@ void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& opts) { throw yexception() << "Can't parse `date-from`, `date-to` parameters"; } } - - // DEBUG - if (timestamp_dev_passed) { - auto timestamp_dev_val = opts.Get("timestamp_deviation"); - ui64 timestamp_deviation; - TryFromString(timestamp_dev_val, timestamp_deviation) && TryFromString(timestamp_dev_val, timestamp_deviation); - Cerr << "timestamp_dev val: " << timestamp_deviation << "\n"; - } - - if (date_from_passed) { - auto date_from_val = opts.Get("date-from"); - ui64 date_from; - TryFromString(date_from_val, date_from) && TryFromString(date_from_val, date_from); - Cerr << "date_from val: " << date_from << "\n"; - } - - if (date_from_passed) { - auto date_to_val = opts.Get("date-to"); - ui64 date_to; - TryFromString(date_to_val, date_to) && TryFromString(date_to_val, date_to); - Cerr << "date_to val: " << date_to << "\n"; - } } THolder TLogWorkloadParams::CreateGenerator() const { diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 34af4890fbef..202bb68c5246 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -14,7 +14,6 @@ class TLogWorkloadParams : public TWorkloadParams { }; void ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) override; - // void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) override; void Parse(const NLastGetopt::TOptsParseResult& opts) override; THolder CreateGenerator() const override; TString GetWorkloadName() const override; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index ff4c3dd8d267..174030fa8327 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -361,17 +361,7 @@ void TWorkloadCommandRun::Config(TConfig& config) { void TWorkloadCommandRun::Parse(TConfig& config) { TWorkloadCommand::Parse(config); - auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation"); - auto date_from_passed = config.ParseResult->Has("date-from"); - auto date_to_passed = config.ParseResult->Has("date-to"); - - Cerr << "____timestamp_dev_passed: " << timestamp_dev_passed << Endl; - Cerr << "____date_from_passed: " << date_from_passed << Endl; - Cerr << "____date_to_passed: " << date_to_passed << Endl; - Params.Parse(ParseResult->GetCommandLineParseResult()); - - // Params.Parse(config); } TWorkloadCommandBase::TWorkloadCommandBase(const TString& name, NYdbWorkload::TWorkloadParams& params, const NYdbWorkload::TWorkloadParams::ECommandType commandType, const TString& description, int type) diff --git a/ydb/public/lib/ydb_cli/common/client_command_options.cpp b/ydb/public/lib/ydb_cli/common/client_command_options.cpp index b6a1c43a47fc..d6932f4d801b 100644 --- a/ydb/public/lib/ydb_cli/common/client_command_options.cpp +++ b/ydb/public/lib/ydb_cli/common/client_command_options.cpp @@ -422,7 +422,6 @@ TOptionsParseResult::TOptionsParseResult(const TClientCommandOptions* options, i , ParseFromCommandLineResult(&options->GetOpts(), argc, argv, throwOnParseError) { for (const auto& clientOption : ClientOptions->ClientOpts) { - // Cerr << clientOption->GetOpt().GetName() << "\n"; if (const auto* optResult = ParseFromCommandLineResult.FindOptParseResult(&clientOption->GetOpt())) { Opts.emplace_back(clientOption, optResult); if (dynamic_cast(clientOption.Get())) { @@ -487,7 +486,7 @@ bool TOptionsParseResult::Has(char name, bool includeDefault) const { return result != nullptr; } -TString TOptionsParseResult::Get(const TString& name, bool includeDefault) const { +const TString& TOptionsParseResult::Get(const TString& name, bool includeDefault) const { if (const TOptionParseResult* result = FindResult(name); result && (includeDefault || result->ValueSource != EOptionValueSource::DefaultValue)) { return result->Values().back(); } diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 68f24fab4943..53460d2b1b7e 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -238,11 +238,6 @@ int TClientCommand::Process(TConfig& config) { void TClientCommand::SaveParseResult(TConfig& config) { Cerr << "ParseResult called\n"; ParseResult = std::make_shared(config.Opts, config.ArgC, (const char**)config.ArgV, config.ThrowOnOptsParseError); - // Cerr << "Opts: ["; - // for (auto x: config.Opts->GetOpts().GetOpts()) { - // Cerr << x->GetName() << ", "; - // } - // Cerr << "]\n"; // Parse options from env and apply default parameters. // Parsing from profiles is only supported at high level commands and occure in ExtractParams() stage. @@ -269,11 +264,7 @@ void TClientCommand::Prepare(TConfig& config) { SetCustomUsage(config); SaveParseResult(config); config.ParseResult = ParseResult.get(); - // try { Parse(config); - // } catch (const std::exception& e) { - // Cerr << "Error occured during Parse() stage: " << e.what(); - // } } void TClientCommand::ExtractParams(TConfig& config) { From 129a758677ec1cd0d67de1d4a933ef0a2ea6d922 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 7 May 2025 20:27:24 +0300 Subject: [PATCH 18/31] passed workload type to parameter validation --- .../abstract/workload_query_generator.h | 2 +- ydb/library/workload/log/log.cpp | 65 ++++++++++++++++++- ydb/library/workload/log/log.h | 2 +- .../lib/ydb_cli/commands/ydb_workload.cpp | 2 +- 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index 1c9516db2a03..a3ed909a4879 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -172,7 +172,7 @@ class TWorkloadParams { virtual ~TWorkloadParams() = default; virtual void ConfigureOpts(NLastGetopt::TOpts& /*opts*/, const ECommandType /*commandType*/, int /*workloadType*/) { }; - virtual void Parse(const NLastGetopt::TOptsParseResult& /*parseResult*/) {}; + virtual void Parse(const NLastGetopt::TOptsParseResult& /*parseResult*/, int /*workloadType*/) {}; virtual THolder CreateGenerator() const = 0; virtual TWorkloadDataInitializer::TList CreateDataInitializers() const { return {}; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index c4ae6b703967..fe3f787eec5c 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -282,6 +282,14 @@ class TRandomLogGenerator { TVector GenerateRandomRows(ui64 count) const { TVector result; result.reserve(count); + + Cerr << "Debug : TimestampDeviation value: " << Params.TimestampStandardDeviationMinutes << "\n"; + if (Params.TimestampDateFrom.has_value()) { + Cerr << "Debug: DateFrom: " << *Params.TimestampDateFrom << "\n"; + } + if (Params.TimestampDateTo.has_value()) { + Cerr << "Debug: DateTo: " << *Params.TimestampDateTo << "\n"; + } for (size_t row = 0; row < count; ++row) { result.emplace_back(); @@ -467,11 +475,44 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT } } -void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& parseResults) { +void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& parseResults, int workloadType) { + // TODO: validate only for this EType's: + // - Insert, + // - Upsert, + // - BulkUpsert + // TODO: 1) check where workloadType is being set + // it's used in RunWorkload and I've checked that it's correct by debug print + // 2) Remove debug logs + + switch (static_cast(workloadType)) { + case TLogGenerator::EType::Insert: + Cerr << "Insert\n"; + break; + case TLogGenerator::EType::Upsert: + Cerr << "Upsert\n"; + break; + case TLogGenerator::EType::BulkUpsert: + Cerr << "BulkUpsert\n"; + break; + case TLogGenerator::EType::Select: + Cerr << "Select\n"; + break; + } + + if (static_cast(workloadType) == TLogGenerator::EType::Select) { + Cerr << "Select: No validation\n"; + return; + } + Cerr << "Not select: validation\n"; + auto timestamp_dev_passed = parseResults.Has("timestamp_deviation"); auto date_from_passed = parseResults.Has("date-from"); auto date_to_passed = parseResults.Has("date-to"); + Cerr << "timestamp_dev_passed: " << timestamp_dev_passed << Endl; + Cerr << "date_from_passed: " << date_from_passed << Endl; + Cerr << "date_to_passed: " << date_to_passed << Endl; + if (!timestamp_dev_passed && (!date_from_passed || !date_to_passed)) { throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; } @@ -497,6 +538,28 @@ void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& parseResults throw yexception() << "Can't parse `date-from`, `date-to` parameters"; } } + + // DEBUG + if (timestamp_dev_passed) { + auto timestamp_dev_val = parseResults.Get("timestamp_deviation"); + ui64 timestamp_deviation; + TryFromString(timestamp_dev_val, timestamp_deviation) && TryFromString(timestamp_dev_val, timestamp_deviation); + Cerr << "timestamp_dev val: " << timestamp_deviation << "\n"; + } + + if (date_from_passed) { + auto date_from_val = parseResults.Get("date-from"); + ui64 date_from; + TryFromString(date_from_val, date_from) && TryFromString(date_from_val, date_from); + Cerr << "date_from val: " << date_from << "\n"; + } + + if (date_from_passed) { + auto date_to_val = parseResults.Get("date-to"); + ui64 date_to; + TryFromString(date_to_val, date_to) && TryFromString(date_to_val, date_to); + Cerr << "date_to val: " << date_to << "\n"; + } } THolder TLogWorkloadParams::CreateGenerator() const { diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 202bb68c5246..f468830473d7 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -14,7 +14,7 @@ class TLogWorkloadParams : public TWorkloadParams { }; void ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) override; - void Parse(const NLastGetopt::TOptsParseResult& opts) override; + void Parse(const NLastGetopt::TOptsParseResult& opts, int wokrloadType) override; THolder CreateGenerator() const override; TString GetWorkloadName() const override; ui64 MinPartitions = 40; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index 174030fa8327..af9451bf0e3a 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -361,7 +361,7 @@ void TWorkloadCommandRun::Config(TConfig& config) { void TWorkloadCommandRun::Parse(TConfig& config) { TWorkloadCommand::Parse(config); - Params.Parse(ParseResult->GetCommandLineParseResult()); + Params.Parse(ParseResult->GetCommandLineParseResult(), Type); } TWorkloadCommandBase::TWorkloadCommandBase(const TString& name, NYdbWorkload::TWorkloadParams& params, const NYdbWorkload::TWorkloadParams::ECommandType commandType, const TString& description, int type) From 837a52f0fe92cf59a2004cd2d909939fdd3ec6a4 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Sat, 10 May 2025 11:33:56 +0300 Subject: [PATCH 19/31] Removed debug log --- ydb/public/lib/ydb_cli/common/command.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 53460d2b1b7e..93053be7bf44 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -236,7 +236,6 @@ int TClientCommand::Process(TConfig& config) { } void TClientCommand::SaveParseResult(TConfig& config) { - Cerr << "ParseResult called\n"; ParseResult = std::make_shared(config.Opts, config.ArgC, (const char**)config.ArgV, config.ThrowOnOptsParseError); // Parse options from env and apply default parameters. From b4a5ead4f2c3019eae981239caa1b09c9a9e61bf Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Sat, 10 May 2025 12:01:37 +0300 Subject: [PATCH 20/31] Removed redundant logs --- ydb/library/workload/log/log.cpp | 59 -------------------------------- 1 file changed, 59 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index fe3f787eec5c..7c2dd19214ad 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -282,14 +282,6 @@ class TRandomLogGenerator { TVector GenerateRandomRows(ui64 count) const { TVector result; result.reserve(count); - - Cerr << "Debug : TimestampDeviation value: " << Params.TimestampStandardDeviationMinutes << "\n"; - if (Params.TimestampDateFrom.has_value()) { - Cerr << "Debug: DateFrom: " << *Params.TimestampDateFrom << "\n"; - } - if (Params.TimestampDateTo.has_value()) { - Cerr << "Debug: DateTo: " << *Params.TimestampDateTo << "\n"; - } for (size_t row = 0; row < count; ++row) { result.emplace_back(); @@ -476,43 +468,14 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT } void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& parseResults, int workloadType) { - // TODO: validate only for this EType's: - // - Insert, - // - Upsert, - // - BulkUpsert - // TODO: 1) check where workloadType is being set - // it's used in RunWorkload and I've checked that it's correct by debug print - // 2) Remove debug logs - - switch (static_cast(workloadType)) { - case TLogGenerator::EType::Insert: - Cerr << "Insert\n"; - break; - case TLogGenerator::EType::Upsert: - Cerr << "Upsert\n"; - break; - case TLogGenerator::EType::BulkUpsert: - Cerr << "BulkUpsert\n"; - break; - case TLogGenerator::EType::Select: - Cerr << "Select\n"; - break; - } - if (static_cast(workloadType) == TLogGenerator::EType::Select) { - Cerr << "Select: No validation\n"; return; } - Cerr << "Not select: validation\n"; auto timestamp_dev_passed = parseResults.Has("timestamp_deviation"); auto date_from_passed = parseResults.Has("date-from"); auto date_to_passed = parseResults.Has("date-to"); - Cerr << "timestamp_dev_passed: " << timestamp_dev_passed << Endl; - Cerr << "date_from_passed: " << date_from_passed << Endl; - Cerr << "date_to_passed: " << date_to_passed << Endl; - if (!timestamp_dev_passed && (!date_from_passed || !date_to_passed)) { throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; } @@ -538,28 +501,6 @@ void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& parseResults throw yexception() << "Can't parse `date-from`, `date-to` parameters"; } } - - // DEBUG - if (timestamp_dev_passed) { - auto timestamp_dev_val = parseResults.Get("timestamp_deviation"); - ui64 timestamp_deviation; - TryFromString(timestamp_dev_val, timestamp_deviation) && TryFromString(timestamp_dev_val, timestamp_deviation); - Cerr << "timestamp_dev val: " << timestamp_deviation << "\n"; - } - - if (date_from_passed) { - auto date_from_val = parseResults.Get("date-from"); - ui64 date_from; - TryFromString(date_from_val, date_from) && TryFromString(date_from_val, date_from); - Cerr << "date_from val: " << date_from << "\n"; - } - - if (date_from_passed) { - auto date_to_val = parseResults.Get("date-to"); - ui64 date_to; - TryFromString(date_to_val, date_to) && TryFromString(date_to_val, date_to); - Cerr << "date_to val: " << date_to << "\n"; - } } THolder TLogWorkloadParams::CreateGenerator() const { From a0d0739f6703cb86f19132097ffa25c6f1275e7d Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Mon, 12 May 2025 17:49:14 +0300 Subject: [PATCH 21/31] Reverted 20 previous commits --- .../abstract/workload_query_generator.h | 1 - ydb/library/workload/log/log.cpp | 57 +------------------ ydb/library/workload/log/log.h | 3 - ydb/library/workload/tpch/driver.h | 6 -- .../lib/ydb_cli/commands/ydb_workload.cpp | 5 -- .../lib/ydb_cli/commands/ydb_workload.h | 1 - 6 files changed, 2 insertions(+), 71 deletions(-) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index a3ed909a4879..71d4a5ad4f5a 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -172,7 +172,6 @@ class TWorkloadParams { virtual ~TWorkloadParams() = default; virtual void ConfigureOpts(NLastGetopt::TOpts& /*opts*/, const ECommandType /*commandType*/, int /*workloadType*/) { }; - virtual void Parse(const NLastGetopt::TOptsParseResult& /*parseResult*/, int /*workloadType*/) {}; virtual THolder CreateGenerator() const = 0; virtual TWorkloadDataInitializer::TList CreateDataInitializers() const { return {}; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 7c2dd19214ad..7339adce9acd 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -1,6 +1,4 @@ #include "log.h" -#include -#include #include #include #include @@ -254,11 +252,6 @@ class TRandomLogGenerator { return result.str(); } - TInstant UniformInstant(ui64 from, ui64 to) const { - TMersenne rnd(Seed()); - return TInstant::FromValue(rnd.Uniform(from, to)); - } - TInstant RandomInstant() const { auto result = TInstant::Now() - TDuration::Seconds(Params.TimestampSubtract); i64 millisecondsDiff = 60 * 1000 * NormalRandom(0., Params.TimestampStandardDeviationMinutes); @@ -286,7 +279,7 @@ class TRandomLogGenerator { for (size_t row = 0; row < count; ++row) { result.emplace_back(); result.back().LogId = CreateGuidAsString().c_str(); - result.back().Ts = Params.TimestampDateFrom.has_value() && Params.TimestampDateTo.has_value() ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant(); + result.back().Ts = RandomInstant(); result.back().Level = RandomNumber(10); result.back().ServiceName = RandomWord(false); result.back().Component = RandomWord(true); @@ -426,17 +419,7 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT opts.AddLongOption("rows", "Number of rows to upsert") .DefaultValue(RowsCnt).StoreResult(&RowsCnt); opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") - .StoreResult(&TimestampStandardDeviationMinutes); - - opts.AddLongOption("date-from", "Left boundary of the interval to generate " - "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " - "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") - .StoreResult(&TimestampDateFrom); - opts.AddLongOption("date-to", "Right boundary of the interval to generate " - "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " - "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") - .StoreResult(&TimestampDateTo); - + .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes); opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") .DefaultValue(0).StoreResult(&TimestampSubtract); opts.AddLongOption("null-percent", "Percent of nulls in generated data") @@ -467,42 +450,6 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT } } -void TLogWorkloadParams::Parse(const NLastGetopt::TOptsParseResult& parseResults, int workloadType) { - if (static_cast(workloadType) == TLogGenerator::EType::Select) { - return; - } - - auto timestamp_dev_passed = parseResults.Has("timestamp_deviation"); - auto date_from_passed = parseResults.Has("date-from"); - auto date_to_passed = parseResults.Has("date-to"); - - if (!timestamp_dev_passed && (!date_from_passed || !date_to_passed)) { - throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; - } - - if (timestamp_dev_passed && (date_from_passed || date_to_passed)) { - throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once"; - } - - if ((date_from_passed && !date_to_passed) || (!date_from_passed && date_to_passed)) { - throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation"; - } - - if (date_from_passed && date_to_passed) { - auto date_from_val = parseResults.Get("date-from"); - auto date_to_val = parseResults.Get("date-to"); - - ui64 date_from, date_to; - if (TryFromString(date_from_val, date_from) && TryFromString(date_to_val, date_to)) { - if (date_from >= date_to) { - throw yexception() << "Invalid interval [`date-from`, `date-to`)"; - } - } else { - throw yexception() << "Can't parse `date-from`, `date-to` parameters"; - } - } -} - THolder TLogWorkloadParams::CreateGenerator() const { return MakeHolder(this); } diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index f468830473d7..7145a0e04791 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -14,7 +14,6 @@ class TLogWorkloadParams : public TWorkloadParams { }; void ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) override; - void Parse(const NLastGetopt::TOptsParseResult& opts, int wokrloadType) override; THolder CreateGenerator() const override; TString GetWorkloadName() const override; ui64 MinPartitions = 40; @@ -25,8 +24,6 @@ class TLogWorkloadParams : public TWorkloadParams { ui64 IntColumnsCnt = 0; ui64 KeyColumnsCnt = 0; ui64 TimestampStandardDeviationMinutes = 0; - std::optional TimestampDateFrom{std::nullopt}; - std::optional TimestampDateTo{std::nullopt}; ui64 TimestampTtlMinutes = 0; ui64 TimestampSubtract = 0; ui64 RowsCnt = 1; diff --git a/ydb/library/workload/tpch/driver.h b/ydb/library/workload/tpch/driver.h index c079a14be5b5..52b3a23ae475 100644 --- a/ydb/library/workload/tpch/driver.h +++ b/ydb/library/workload/tpch/driver.h @@ -6,13 +6,7 @@ extern "C" { #include #include -#ifdef DT_CHR -#undef DT_CHR -#endif #include -#ifdef DT_CHR -#undef DT_CHR -#endif #include void InitTpchGen(DSS_HUGE scale); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index af9451bf0e3a..a5ce8a927349 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -359,11 +359,6 @@ void TWorkloadCommandRun::Config(TConfig& config) { Params.ConfigureOpts(config.Opts->GetOpts(), NYdbWorkload::TWorkloadParams::ECommandType::Run, Type); } -void TWorkloadCommandRun::Parse(TConfig& config) { - TWorkloadCommand::Parse(config); - Params.Parse(ParseResult->GetCommandLineParseResult(), Type); -} - TWorkloadCommandBase::TWorkloadCommandBase(const TString& name, NYdbWorkload::TWorkloadParams& params, const NYdbWorkload::TWorkloadParams::ECommandType commandType, const TString& description, int type) : TYdbCommand(name, std::initializer_list(), description) , CommandType(commandType) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.h b/ydb/public/lib/ydb_cli/commands/ydb_workload.h index ece29031e546..909e6a72a5fe 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.h @@ -73,7 +73,6 @@ class TWorkloadCommandRun : public TWorkloadCommand { public: TWorkloadCommandRun(NYdbWorkload::TWorkloadParams& params, const NYdbWorkload::IWorkloadQueryGenerator::TWorkloadType& workload); virtual void Config(TConfig& config) override; - virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; private: From 05aa423002894e02a0787c740f2550809eef7f5f Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Sun, 11 May 2025 21:31:38 +0300 Subject: [PATCH 22/31] uniform Ts --- ydb/library/workload/log/log.cpp | 48 +++++++++++++++++++++++++++++++- ydb/library/workload/log/log.h | 2 ++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 7339adce9acd..c3b91ede4f9e 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -252,6 +254,11 @@ class TRandomLogGenerator { return result.str(); } + TInstant UniformInstant(ui64 from, ui64 to) const { + TMersenne rnd(Seed()); + return TInstant::FromValue(rnd.Uniform(from, to)); + } + TInstant RandomInstant() const { auto result = TInstant::Now() - TDuration::Seconds(Params.TimestampSubtract); i64 millisecondsDiff = 60 * 1000 * NormalRandom(0., Params.TimestampStandardDeviationMinutes); @@ -267,19 +274,42 @@ class TRandomLogGenerator { return RandomNumber(100) >= Params.NullPercent; } + void CheckParams() const { + const bool timestampDevPassed = Params.TimestampStandardDeviationMinutes; + const bool dateFromPassed = !!Params.TimestampDateFrom; + const bool dateToPassed = !!Params.TimestampDateTo; + + if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) { + throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; + } + + if (timestampDevPassed && (dateFromPassed || dateToPassed)) { + throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once"; + } + + if ((dateFromPassed && !dateToPassed) || (!dateFromPassed && dateToPassed)) { + throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation"; + } + + if (dateFromPassed && dateToPassed && *Params.TimestampDateFrom >= *Params.TimestampDateTo) { + throw yexception() << "Invalid interval [`date-from`, `date-to`)"; + } + } + public: explicit TRandomLogGenerator(const TLogWorkloadParams& params) : Params(params) {} TVector GenerateRandomRows(ui64 count) const { + CheckParams(); TVector result; result.reserve(count); for (size_t row = 0; row < count; ++row) { result.emplace_back(); result.back().LogId = CreateGuidAsString().c_str(); - result.back().Ts = RandomInstant(); + result.back().Ts = !!Params.TimestampDateFrom && !!Params.TimestampDateTo ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant(); result.back().Level = RandomNumber(10); result.back().ServiceName = RandomWord(false); result.back().Component = RandomWord(true); @@ -420,6 +450,14 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT .DefaultValue(RowsCnt).StoreResult(&RowsCnt); opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes); + opts.AddLongOption("date-from", "Left boundary of the interval to generate " + "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " + "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") + .StoreResult(&TimestampDateFrom); + opts.AddLongOption("date-to", "Right boundary of the interval to generate " + "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " + "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") + .StoreResult(&TimestampDateTo); opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") .DefaultValue(0).StoreResult(&TimestampSubtract); opts.AddLongOption("null-percent", "Percent of nulls in generated data") @@ -442,6 +480,14 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT .DefaultValue(RowsCnt).StoreResult(&RowsCnt); opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes); + opts.AddLongOption("date-from", "Left boundary of the interval to generate " + "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " + "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") + .StoreResult(&TimestampDateFrom); + opts.AddLongOption("date-to", "Right boundary of the interval to generate " + "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " + "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") + .StoreResult(&TimestampDateTo); opts.AddLongOption("null-percent", "Percent of nulls in generated data") .DefaultValue(NullPercent).StoreResult(&NullPercent); break; diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 7145a0e04791..8649de8fa56b 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -24,6 +24,8 @@ class TLogWorkloadParams : public TWorkloadParams { ui64 IntColumnsCnt = 0; ui64 KeyColumnsCnt = 0; ui64 TimestampStandardDeviationMinutes = 0; + TMaybe TimestampDateFrom; + TMaybe TimestampDateTo; ui64 TimestampTtlMinutes = 0; ui64 TimestampSubtract = 0; ui64 RowsCnt = 1; From 43f1f5829bbfa7eee54c740dc80ccb731c726c96 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Sun, 11 May 2025 21:44:42 +0300 Subject: [PATCH 23/31] refactor params --- ydb/library/workload/log/log.cpp | 88 ++++++++++++-------------------- ydb/library/workload/log/log.h | 4 ++ 2 files changed, 38 insertions(+), 54 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index c3b91ede4f9e..9b8968d1a3de 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -390,6 +390,37 @@ TQueryInfoList TLogGenerator::GetWorkload(int type) { } } +void TLogWorkloadParams::ConfigureOptsColumns(NLastGetopt::TOpts& opts) { + opts.AddLongOption("int-cols", "Number of int columns") + .DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt); + opts.AddLongOption("str-cols", "Number of string columns") + .DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt); + opts.AddLongOption("key-cols", "Number of key columns") + .DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt); +} + +void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) { + opts.AddLongOption("len", "String len") + .DefaultValue(StringLen).StoreResult(&StringLen); + ConfigureOptsColumns(opts); + opts.AddLongOption("rows", "Number of rows to upsert") + .DefaultValue(RowsCnt).StoreResult(&RowsCnt); + opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") + .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes); + opts.AddLongOption("date-from", "Left boundary of the interval to generate " + "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " + "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") + .StoreResult(&TimestampDateFrom); + opts.AddLongOption("date-to", "Right boundary of the interval to generate " + "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " + "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") + .StoreResult(&TimestampDateTo); + opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") + .DefaultValue(0).StoreResult(&TimestampSubtract); + opts.AddLongOption("null-percent", "Percent of nulls in generated data") + .DefaultValue(NullPercent).StoreResult(&NullPercent); +} + void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) { opts.AddLongOption('p', "path", "Path where benchmark tables are located") .Optional() @@ -409,14 +440,7 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT .DefaultValue(PartitionSizeMb).StoreResult(&PartitionSizeMb); opts.AddLongOption("auto-partition", "Enable auto partitioning by load.") .DefaultValue(PartitionsByLoad).StoreResult(&PartitionsByLoad); - opts.AddLongOption("len", "String len") - .DefaultValue(StringLen).StoreResult(&StringLen); - opts.AddLongOption("int-cols", "Number of int columns") - .DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt); - opts.AddLongOption("str-cols", "Number of string columns") - .DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt); - opts.AddLongOption("key-cols", "Number of key columns") - .DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt); + ConfigureOptsColumns(opts); opts.AddLongOption("ttl", "TTL for timestamp column in minutes") .DefaultValue(TimestampTtlMinutes).StoreResult(&TimestampTtlMinutes); opts.AddLongOption("store", "Storage type." @@ -438,58 +462,14 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT case TLogGenerator::EType::Insert: case TLogGenerator::EType::Upsert: case TLogGenerator::EType::BulkUpsert: - opts.AddLongOption("len", "String len") - .DefaultValue(StringLen).StoreResult(&StringLen); - opts.AddLongOption("int-cols", "Number of int columns") - .DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt); - opts.AddLongOption("str-cols", "Number of string columns") - .DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt); - opts.AddLongOption("key-cols", "Number of key columns") - .DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt); - opts.AddLongOption("rows", "Number of rows to upsert") - .DefaultValue(RowsCnt).StoreResult(&RowsCnt); - opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") - .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes); - opts.AddLongOption("date-from", "Left boundary of the interval to generate " - "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " - "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") - .StoreResult(&TimestampDateFrom); - opts.AddLongOption("date-to", "Right boundary of the interval to generate " - "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " - "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") - .StoreResult(&TimestampDateTo); - opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted") - .DefaultValue(0).StoreResult(&TimestampSubtract); - opts.AddLongOption("null-percent", "Percent of nulls in generated data") - .DefaultValue(NullPercent).StoreResult(&NullPercent); + ConfigureOptsFillData(opts); break; case TLogGenerator::EType::Select: break; } break; case TWorkloadParams::ECommandType::Import: - opts.AddLongOption("len", "String len") - .DefaultValue(StringLen).StoreResult(&StringLen); - opts.AddLongOption("int-cols", "Number of int columns") - .DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt); - opts.AddLongOption("str-cols", "Number of string columns") - .DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt); - opts.AddLongOption("key-cols", "Number of key columns") - .DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt); - opts.AddLongOption("rows", "Number of rows to upsert") - .DefaultValue(RowsCnt).StoreResult(&RowsCnt); - opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") - .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes); - opts.AddLongOption("date-from", "Left boundary of the interval to generate " - "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " - "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") - .StoreResult(&TimestampDateFrom); - opts.AddLongOption("date-to", "Right boundary of the interval to generate " - "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' " - "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") - .StoreResult(&TimestampDateTo); - opts.AddLongOption("null-percent", "Percent of nulls in generated data") - .DefaultValue(NullPercent).StoreResult(&NullPercent); + ConfigureOptsFillData(opts); break; default: break; diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 8649de8fa56b..82638cbb334b 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -36,6 +36,10 @@ class TLogWorkloadParams : public TWorkloadParams { YDB_READONLY(EStoreType, StoreType, EStoreType::Row); TWorkloadDataInitializer::TList CreateDataInitializers() const override; + +private: + void ConfigureOptsFillData(NLastGetopt::TOpts& opts); + void ConfigureOptsColumns(NLastGetopt::TOpts& opts); }; class TLogGenerator final: public TWorkloadQueryGeneratorBase { From df4a93a79b422863a714036aa82967674e1e1fea Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 12 May 2025 00:10:47 +0300 Subject: [PATCH 24/31] fix --- ydb/library/workload/log/log.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 9b8968d1a3de..60b71346217e 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -391,6 +391,8 @@ TQueryInfoList TLogGenerator::GetWorkload(int type) { } void TLogWorkloadParams::ConfigureOptsColumns(NLastGetopt::TOpts& opts) { + opts.AddLongOption("len", "String len") + .DefaultValue(StringLen).StoreResult(&StringLen); opts.AddLongOption("int-cols", "Number of int columns") .DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt); opts.AddLongOption("str-cols", "Number of string columns") @@ -400,8 +402,6 @@ void TLogWorkloadParams::ConfigureOptsColumns(NLastGetopt::TOpts& opts) { } void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) { - opts.AddLongOption("len", "String len") - .DefaultValue(StringLen).StoreResult(&StringLen); ConfigureOptsColumns(opts); opts.AddLongOption("rows", "Number of rows to upsert") .DefaultValue(RowsCnt).StoreResult(&RowsCnt); From f9194b63d410450a109865533679668bd702279d Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Tue, 13 May 2025 15:07:32 +0300 Subject: [PATCH 25/31] Moved params validation before workload run --- ydb/library/workload/log/log.cpp | 28 ++++++++++++++++++- ydb/library/workload/log/log.h | 1 + .../lib/ydb_cli/commands/ydb_workload.cpp | 10 ++++++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 60b71346217e..29e1a7cbc40e 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -302,7 +302,7 @@ class TRandomLogGenerator { {} TVector GenerateRandomRows(ui64 count) const { - CheckParams(); + // CheckParams(); TVector result; result.reserve(count); @@ -421,6 +421,30 @@ void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) { .DefaultValue(NullPercent).StoreResult(&NullPercent); } +void TLogWorkloadParams::Validate() const { + const bool timestampDevPassed = TimestampStandardDeviationMinutes; + const bool dateFromPassed = !!TimestampDateFrom; + const bool dateToPassed = !!TimestampDateTo; + + if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) { + throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; + } + + if (timestampDevPassed && (dateFromPassed || dateToPassed)) { + throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once"; + } + + if ((dateFromPassed && !dateToPassed) || (!dateFromPassed && dateToPassed)) { + throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation"; + } + + if (dateFromPassed && dateToPassed && *TimestampDateFrom >= *TimestampDateTo) { + throw yexception() << "Invalid interval [`date-from`, `date-to`)"; + } + + return; +} + void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) { opts.AddLongOption('p', "path", "Path where benchmark tables are located") .Optional() @@ -477,6 +501,8 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT } THolder TLogWorkloadParams::CreateGenerator() const { + Validate(); + return MakeHolder(this); } diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 82638cbb334b..83e19c754195 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -38,6 +38,7 @@ class TLogWorkloadParams : public TWorkloadParams { TWorkloadDataInitializer::TList CreateDataInitializers() const override; private: + void Validate() const; void ConfigureOptsFillData(NLastGetopt::TOpts& opts); void ConfigureOptsColumns(NLastGetopt::TOpts& opts); }; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index a5ce8a927349..b7df67c274b0 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -216,7 +216,12 @@ void TWorkloadCommand::WorkerFn(int taskId, NYdbWorkload::IWorkloadQueryGenerato }; while (Now() < StopTime) { - auto queryInfoList = workloadGen.GetWorkload(type); + NYdbWorkload::TQueryInfoList queryInfoList; + try { + queryInfoList = workloadGen.GetWorkload(type); + } catch (yexception& exception) { + return; + } if (queryInfoList.empty()) { Cerr << "Task ID: " << taskId << ". No queries to run." << Endl; return; @@ -279,6 +284,9 @@ int TWorkloadCommand::RunWorkload(NYdbWorkload::IWorkloadQueryGenerator& workloa StopTime = StartTime + TDuration::Seconds(TotalSec); NPar::LocalExecutor().RunAdditionalThreads(Threads); + // bool invalidParams = false; + // До разделения на потоки сделать валидацию. + auto futures = NPar::LocalExecutor().ExecRangeWithFutures([this, &workloadGen, type](int id) { try { WorkerFn(id, workloadGen, type); From e36fa044b8b8f18f4144acc58dcc7ab9a9fbe1f8 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Tue, 13 May 2025 16:24:40 +0300 Subject: [PATCH 26/31] Removed redundant try/catch --- ydb/public/lib/ydb_cli/commands/ydb_workload.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index b7df67c274b0..66a98b3fc6e9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -217,11 +217,7 @@ void TWorkloadCommand::WorkerFn(int taskId, NYdbWorkload::IWorkloadQueryGenerato while (Now() < StopTime) { NYdbWorkload::TQueryInfoList queryInfoList; - try { - queryInfoList = workloadGen.GetWorkload(type); - } catch (yexception& exception) { - return; - } + queryInfoList = workloadGen.GetWorkload(type); if (queryInfoList.empty()) { Cerr << "Task ID: " << taskId << ". No queries to run." << Endl; return; From db41c939a5bf67bdb3b20afe7a1a536308486d64 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Wed, 14 May 2025 11:48:21 +0300 Subject: [PATCH 27/31] Debug logs --- ydb/library/workload/log/log.cpp | 4 ++++ ydb/public/lib/ydb_cli/commands/ydb_workload.cpp | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 29e1a7cbc40e..e56a0a9d6040 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -426,6 +426,10 @@ void TLogWorkloadParams::Validate() const { const bool dateFromPassed = !!TimestampDateFrom; const bool dateToPassed = !!TimestampDateTo; + Cerr << "TimestampDevPassed: " << timestampDevPassed << "\n"; + Cerr << "DateFromPassed: " << dateFromPassed << "\n"; + Cerr << "DateToPassed: " << dateToPassed << "\n"; + if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) { throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index 66a98b3fc6e9..5b0eef057bc4 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -216,8 +216,7 @@ void TWorkloadCommand::WorkerFn(int taskId, NYdbWorkload::IWorkloadQueryGenerato }; while (Now() < StopTime) { - NYdbWorkload::TQueryInfoList queryInfoList; - queryInfoList = workloadGen.GetWorkload(type); + auto queryInfoList = workloadGen.GetWorkload(type); if (queryInfoList.empty()) { Cerr << "Task ID: " << taskId << ". No queries to run." << Endl; return; From 65cb9c80189cc2f1fdda61e6de0305f4b4572061 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Thu, 15 May 2025 12:41:37 +0300 Subject: [PATCH 28/31] Moved validation before running workload in several threads --- .../abstract/workload_query_generator.h | 1 + ydb/library/workload/log/log.cpp | 27 +------------------ ydb/library/workload/log/log.h | 2 +- .../lib/ydb_cli/commands/ydb_workload.cpp | 5 ++-- 4 files changed, 5 insertions(+), 30 deletions(-) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index 71d4a5ad4f5a..a8d9c26b914c 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -178,6 +178,7 @@ class TWorkloadParams { } virtual TString GetWorkloadName() const = 0; + virtual void Validate() const {}; public: ui64 BulkSize = 10000; std::string DbPath; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index e56a0a9d6040..290321986f88 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -274,35 +274,12 @@ class TRandomLogGenerator { return RandomNumber(100) >= Params.NullPercent; } - void CheckParams() const { - const bool timestampDevPassed = Params.TimestampStandardDeviationMinutes; - const bool dateFromPassed = !!Params.TimestampDateFrom; - const bool dateToPassed = !!Params.TimestampDateTo; - - if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) { - throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; - } - - if (timestampDevPassed && (dateFromPassed || dateToPassed)) { - throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once"; - } - - if ((dateFromPassed && !dateToPassed) || (!dateFromPassed && dateToPassed)) { - throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation"; - } - - if (dateFromPassed && dateToPassed && *Params.TimestampDateFrom >= *Params.TimestampDateTo) { - throw yexception() << "Invalid interval [`date-from`, `date-to`)"; - } - } - public: explicit TRandomLogGenerator(const TLogWorkloadParams& params) : Params(params) {} TVector GenerateRandomRows(ui64 count) const { - // CheckParams(); TVector result; result.reserve(count); @@ -445,7 +422,7 @@ void TLogWorkloadParams::Validate() const { if (dateFromPassed && dateToPassed && *TimestampDateFrom >= *TimestampDateTo) { throw yexception() << "Invalid interval [`date-from`, `date-to`)"; } - + return; } @@ -505,8 +482,6 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT } THolder TLogWorkloadParams::CreateGenerator() const { - Validate(); - return MakeHolder(this); } diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 83e19c754195..24e975508819 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -37,8 +37,8 @@ class TLogWorkloadParams : public TWorkloadParams { YDB_READONLY(EStoreType, StoreType, EStoreType::Row); TWorkloadDataInitializer::TList CreateDataInitializers() const override; + void Validate() const override; private: - void Validate() const; void ConfigureOptsFillData(NLastGetopt::TOpts& opts); void ConfigureOptsColumns(NLastGetopt::TOpts& opts); }; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index 5b0eef057bc4..588c34c8c5d7 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -279,9 +279,7 @@ int TWorkloadCommand::RunWorkload(NYdbWorkload::IWorkloadQueryGenerator& workloa StopTime = StartTime + TDuration::Seconds(TotalSec); NPar::LocalExecutor().RunAdditionalThreads(Threads); - // bool invalidParams = false; - // До разделения на потоки сделать валидацию. - + auto futures = NPar::LocalExecutor().ExecRangeWithFutures([this, &workloadGen, type](int id) { try { WorkerFn(id, workloadGen, type); @@ -353,6 +351,7 @@ int TWorkloadCommandRun::Run(TConfig& config) { PrepareForRun(config); Params.DbPath = config.Database; auto workloadGen = Params.CreateGenerator(); + Params.Validate(); return RunWorkload(*workloadGen, Type); } From adae9e5f13b3f0a92cd53e6a9944c1b164182c78 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Fri, 16 May 2025 16:53:34 +0300 Subject: [PATCH 29/31] Disabled validation for select --- ydb/library/workload/abstract/workload_query_generator.h | 2 +- ydb/library/workload/log/log.cpp | 6 +++++- ydb/library/workload/log/log.h | 2 +- ydb/public/lib/ydb_cli/commands/ydb_workload.cpp | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index a8d9c26b914c..28bf4c412685 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -178,7 +178,7 @@ class TWorkloadParams { } virtual TString GetWorkloadName() const = 0; - virtual void Validate() const {}; + virtual void Validate(int /*workloadType*/) const {}; public: ui64 BulkSize = 10000; std::string DbPath; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index 290321986f88..a459ec0033f4 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -398,7 +398,11 @@ void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) { .DefaultValue(NullPercent).StoreResult(&NullPercent); } -void TLogWorkloadParams::Validate() const { +void TLogWorkloadParams::Validate(int workloadType) const { + if (static_cast(workloadType) == TLogGenerator::EType::Select) { + return; + } + const bool timestampDevPassed = TimestampStandardDeviationMinutes; const bool dateFromPassed = !!TimestampDateFrom; const bool dateToPassed = !!TimestampDateTo; diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 24e975508819..e42535d00693 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -37,7 +37,7 @@ class TLogWorkloadParams : public TWorkloadParams { YDB_READONLY(EStoreType, StoreType, EStoreType::Row); TWorkloadDataInitializer::TList CreateDataInitializers() const override; - void Validate() const override; + void Validate(int workloadType) const override; private: void ConfigureOptsFillData(NLastGetopt::TOpts& opts); void ConfigureOptsColumns(NLastGetopt::TOpts& opts); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index 588c34c8c5d7..ea789eeeee8d 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -351,7 +351,7 @@ int TWorkloadCommandRun::Run(TConfig& config) { PrepareForRun(config); Params.DbPath = config.Database; auto workloadGen = Params.CreateGenerator(); - Params.Validate(); + Params.Validate(Type); return RunWorkload(*workloadGen, Type); } From 60377dfc41b74bdf1b6d26cd0ac4bc37bfcbea4f Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Sat, 17 May 2025 20:35:32 +0300 Subject: [PATCH 30/31] Added workloadType and ECommandType to workload params validation --- .../abstract/workload_query_generator.h | 2 +- ydb/library/workload/log/log.cpp | 61 ++++++++++++------- ydb/library/workload/log/log.h | 2 +- .../lib/ydb_cli/commands/ydb_workload.cpp | 2 +- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index 28bf4c412685..5879b6f24c90 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -178,7 +178,7 @@ class TWorkloadParams { } virtual TString GetWorkloadName() const = 0; - virtual void Validate(int /*workloadType*/) const {}; + virtual void Validate(const ECommandType /*commandType*/, int /*workloadType*/) const {}; public: ui64 BulkSize = 10000; std::string DbPath; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index a459ec0033f4..e4323b070c82 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -398,35 +398,54 @@ void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) { .DefaultValue(NullPercent).StoreResult(&NullPercent); } -void TLogWorkloadParams::Validate(int workloadType) const { - if (static_cast(workloadType) == TLogGenerator::EType::Select) { - return; - } - +void TLogWorkloadParams::Validate(const ECommandType commandType, int workloadType) const { const bool timestampDevPassed = TimestampStandardDeviationMinutes; const bool dateFromPassed = !!TimestampDateFrom; const bool dateToPassed = !!TimestampDateTo; - Cerr << "TimestampDevPassed: " << timestampDevPassed << "\n"; - Cerr << "DateFromPassed: " << dateFromPassed << "\n"; - Cerr << "DateToPassed: " << dateToPassed << "\n"; - - if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) { - throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; - } + switch (commandType) { + case TWorkloadParams::ECommandType::Init: + break; + case TWorkloadParams::ECommandType::Run: + switch (static_cast(workloadType)) { + case TLogGenerator::EType::Insert: + case TLogGenerator::EType::Upsert: + case TLogGenerator::EType::BulkUpsert: + + + Cerr << "TimestampDevPassed: " << timestampDevPassed << "\n"; + Cerr << "DateFromPassed: " << dateFromPassed << "\n"; + Cerr << "DateToPassed: " << dateToPassed << "\n"; + + if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) { + throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; + } - if (timestampDevPassed && (dateFromPassed || dateToPassed)) { - throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once"; - } + if (timestampDevPassed && (dateFromPassed || dateToPassed)) { + throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once"; + } - if ((dateFromPassed && !dateToPassed) || (!dateFromPassed && dateToPassed)) { - throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation"; - } + if ((dateFromPassed && !dateToPassed) || (!dateFromPassed && dateToPassed)) { + throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation"; + } - if (dateFromPassed && dateToPassed && *TimestampDateFrom >= *TimestampDateTo) { - throw yexception() << "Invalid interval [`date-from`, `date-to`)"; + if (dateFromPassed && dateToPassed && *TimestampDateFrom >= *TimestampDateTo) { + throw yexception() << "Invalid interval [`date-from`, `date-to`)"; + } + + break; + case TLogGenerator::EType::Select: + + break; + } + break; + case TWorkloadParams::ECommandType::Clean: + break; + case TWorkloadParams::ECommandType::Root: + break; + case TWorkloadParams::ECommandType::Import: + break; } - return; } diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index e42535d00693..74a8fe810f88 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -37,7 +37,7 @@ class TLogWorkloadParams : public TWorkloadParams { YDB_READONLY(EStoreType, StoreType, EStoreType::Row); TWorkloadDataInitializer::TList CreateDataInitializers() const override; - void Validate(int workloadType) const override; + void Validate(const ECommandType commandType, int workloadType) const override; private: void ConfigureOptsFillData(NLastGetopt::TOpts& opts); void ConfigureOptsColumns(NLastGetopt::TOpts& opts); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp index ea789eeeee8d..341906e6446f 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -351,7 +351,7 @@ int TWorkloadCommandRun::Run(TConfig& config) { PrepareForRun(config); Params.DbPath = config.Database; auto workloadGen = Params.CreateGenerator(); - Params.Validate(Type); + Params.Validate(NYdbWorkload::TWorkloadParams::ECommandType::Run, Type); return RunWorkload(*workloadGen, Type); } From 36e1bca001b1a2ec89ec5221d1bbd62efc57cb84 Mon Sep 17 00:00:00 2001 From: Emil Garipov Date: Sat, 17 May 2025 21:26:13 +0300 Subject: [PATCH 31/31] Added default logic for timestamp_deviation param --- .../abstract/workload_query_generator.h | 2 +- ydb/library/workload/log/log.cpp | 24 +++++++++---------- ydb/library/workload/log/log.h | 4 ++-- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/ydb/library/workload/abstract/workload_query_generator.h b/ydb/library/workload/abstract/workload_query_generator.h index 5879b6f24c90..7db3ae1bb258 100644 --- a/ydb/library/workload/abstract/workload_query_generator.h +++ b/ydb/library/workload/abstract/workload_query_generator.h @@ -178,7 +178,7 @@ class TWorkloadParams { } virtual TString GetWorkloadName() const = 0; - virtual void Validate(const ECommandType /*commandType*/, int /*workloadType*/) const {}; + virtual void Validate(const ECommandType /*commandType*/, int /*workloadType*/) {}; public: ui64 BulkSize = 10000; std::string DbPath; diff --git a/ydb/library/workload/log/log.cpp b/ydb/library/workload/log/log.cpp index e4323b070c82..2116a57650e9 100644 --- a/ydb/library/workload/log/log.cpp +++ b/ydb/library/workload/log/log.cpp @@ -261,7 +261,11 @@ class TRandomLogGenerator { TInstant RandomInstant() const { auto result = TInstant::Now() - TDuration::Seconds(Params.TimestampSubtract); - i64 millisecondsDiff = 60 * 1000 * NormalRandom(0., Params.TimestampStandardDeviationMinutes); + ui64 timestampStandardDeviationMinutes = 0; + if (Params.TimestampStandardDeviationMinutes.Defined()) { + timestampStandardDeviationMinutes = *Params.TimestampStandardDeviationMinutes; + } + i64 millisecondsDiff = 60 * 1000 * NormalRandom(0., timestampStandardDeviationMinutes); if (millisecondsDiff >= 0) { // TDuration::MilliSeconds can't be negative for some reason... result += TDuration::MilliSeconds(millisecondsDiff); } else { @@ -383,7 +387,7 @@ void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) { opts.AddLongOption("rows", "Number of rows to upsert") .DefaultValue(RowsCnt).StoreResult(&RowsCnt); opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.") - .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes); + .StoreResult(&TimestampStandardDeviationMinutes); opts.AddLongOption("date-from", "Left boundary of the interval to generate " "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' " "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'") @@ -398,8 +402,8 @@ void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) { .DefaultValue(NullPercent).StoreResult(&NullPercent); } -void TLogWorkloadParams::Validate(const ECommandType commandType, int workloadType) const { - const bool timestampDevPassed = TimestampStandardDeviationMinutes; +void TLogWorkloadParams::Validate(const ECommandType commandType, int workloadType) { + bool timestampDevPassed = !!TimestampStandardDeviationMinutes; const bool dateFromPassed = !!TimestampDateFrom; const bool dateToPassed = !!TimestampDateTo; @@ -411,14 +415,9 @@ void TLogWorkloadParams::Validate(const ECommandType commandType, int workloadTy case TLogGenerator::EType::Insert: case TLogGenerator::EType::Upsert: case TLogGenerator::EType::BulkUpsert: - - - Cerr << "TimestampDevPassed: " << timestampDevPassed << "\n"; - Cerr << "DateFromPassed: " << dateFromPassed << "\n"; - Cerr << "DateToPassed: " << dateToPassed << "\n"; - - if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) { - throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to"; + if (!timestampDevPassed && !dateFromPassed && !dateToPassed) { + timestampDevPassed = true; + TimestampStandardDeviationMinutes = 0; } if (timestampDevPassed && (dateFromPassed || dateToPassed)) { @@ -435,7 +434,6 @@ void TLogWorkloadParams::Validate(const ECommandType commandType, int workloadTy break; case TLogGenerator::EType::Select: - break; } break; diff --git a/ydb/library/workload/log/log.h b/ydb/library/workload/log/log.h index 74a8fe810f88..5a4f900ce61b 100644 --- a/ydb/library/workload/log/log.h +++ b/ydb/library/workload/log/log.h @@ -23,7 +23,7 @@ class TLogWorkloadParams : public TWorkloadParams { ui64 StrColumnsCnt = 0; ui64 IntColumnsCnt = 0; ui64 KeyColumnsCnt = 0; - ui64 TimestampStandardDeviationMinutes = 0; + TMaybe TimestampStandardDeviationMinutes; TMaybe TimestampDateFrom; TMaybe TimestampDateTo; ui64 TimestampTtlMinutes = 0; @@ -37,7 +37,7 @@ class TLogWorkloadParams : public TWorkloadParams { YDB_READONLY(EStoreType, StoreType, EStoreType::Row); TWorkloadDataInitializer::TList CreateDataInitializers() const override; - void Validate(const ECommandType commandType, int workloadType) const override; + void Validate(const ECommandType commandType, int workloadType) override; private: void ConfigureOptsFillData(NLastGetopt::TOpts& opts); void ConfigureOptsColumns(NLastGetopt::TOpts& opts);