Skip to content

Commit cb98963

Browse files
shnikdshnikd
authored
Merge 021801d into e6a4bbf 
2 parents e6a4bbf + 021801d commit cb98963

File tree

13 files changed

+460
-114
lines changed

13 files changed

+460
-114
lines changed

‎ydb/core/kqp/common/simple/temp_tables.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,21 @@
22

33
namespace NKikimr::NKqp {
44

5+
THashMap<TString, TKqpTempTablesState::TTempTableInfo>::const_iterator
6+
TKqpTempTablesState::FindInfo(const std::string_view& path, bool withSessionId) const {
7+
if (!withSessionId) {
8+
return TempTables.find(path);
9+
}
10+
11+
if (path.size() < SessionId.size()) {
12+
return TempTables.end();
13+
}
14+
size_t pos = path.size() - SessionId.size();
15+
if (path.substr(pos) != SessionId) {
16+
return TempTables.end();
17+
}
18+
19+
return TempTables.find(path.substr(0, pos));
20+
}
21+
522
} // namespace NKikimr::NKqp

‎ydb/core/kqp/common/simple/temp_tables.h‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include <ydb/library/aclib/aclib.h>
44

55
#include <optional>
6+
#include <string_view>
67

78
#include <util/generic/fwd.h>
89
#include <util/generic/hash.h>
@@ -14,14 +15,15 @@ struct TKqpTempTablesState {
1415
struct TTempTableInfo {
1516
TString Name;
1617
TString WorkingDir;
17-
TString Database;
1818
TIntrusiveConstPtr<NACLib::TUserToken> UserToken;
19-
TString Cluster;
2019
};
21-
std::optional<TString> SessionId;
22-
THashMap<std::pair<TString, TString>, TTempTableInfo> TempTables;
20+
TString SessionId;
21+
THashMap<TString, TTempTableInfo> TempTables;
2322

2423
using TConstPtr = std::shared_ptr<const TKqpTempTablesState>;
24+
25+
THashMap<TString, TTempTableInfo>::const_iterator
26+
FindInfo(const std::string_view& path, bool withSessionId = false) const;
2527
};
2628

2729
} // namespace NKikimr::NKqp

‎ydb/core/kqp/compile_service/kqp_compile_service.cpp‎

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,9 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
567567
Counters->ReportCompileRequestGet(dbCounters);
568568

569569
auto compileResult = QueryCache.FindByUid(*request.Uid, request.KeepInCache);
570+
if (HasTempTablesNameClashes(compileResult, request.TempTablesState)) {
571+
compileResult = nullptr;
572+
}
570573
if (compileResult) {
571574
Y_ENSURE(compileResult->Query);
572575
if (compileResult->Query->UserSid == userSid) {
@@ -610,6 +613,10 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
610613
}
611614

612615
auto compileResult = QueryCache.FindByQuery(query, request.KeepInCache);
616+
if (HasTempTablesNameClashes(compileResult, request.TempTablesState)) {
617+
compileResult = nullptr;
618+
}
619+
613620
if (compileResult) {
614621
Counters->ReportQueryCacheHit(dbCounters, true);
615622

@@ -672,7 +679,11 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
672679
auto dbCounters = request.DbCounters;
673680
Counters->ReportRecompileRequestGet(dbCounters);
674681

675-
auto compileResult = QueryCache.FindByUid(request.Uid, false);
682+
TKqpCompileResult::TConstPtr compileResult = QueryCache.FindByUid(request.Uid, false);
683+
if (HasTempTablesNameClashes(compileResult, request.TempTablesState)) {
684+
compileResult = nullptr;
685+
}
686+
676687
if (compileResult || request.Query) {
677688
Counters->ReportCompileRequestCompile(dbCounters);
678689

@@ -736,19 +747,12 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
736747

737748
bool keepInCache = compileRequest.KeepInCache && compileResult->AllowCache;
738749

750+
bool hasTempTablesNameClashes = HasTempTablesNameClashes(compileResult, compileRequest.TempTablesState, true);
751+
739752
try {
740753
if (compileResult->Status == Ydb::StatusIds::SUCCESS) {
741-
if (QueryCache.FindByUid(compileResult->Uid, false)) {
742-
QueryCache.Replace(compileResult);
743-
} else if (keepInCache) {
744-
if (QueryCache.Insert(compileResult, TableServiceConfig.GetEnableAstCache())) {
745-
Counters->CompileQueryCacheEvicted->Inc();
746-
}
747-
if (compileResult->Query && compileResult->Query->Settings.IsPrepareQuery) {
748-
if (InsertPreparingQuery(compileResult, compileRequest.KeepInCache)) {
749-
Counters->CompileQueryCacheEvicted->Inc();
750-
};
751-
}
754+
if (!hasTempTablesNameClashes) {
755+
UpdateQueryCache(compileResult, keepInCache);
752756
}
753757

754758
if (ev->Get()->ReplayMessage) {
@@ -762,8 +766,10 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
762766
request.Cookie, std::move(request.Orbit), std::move(request.CompileServiceSpan), (CollectDiagnostics ? ev->Get()->ReplayMessageUserView : std::nullopt));
763767
}
764768
} else {
765-
if (QueryCache.FindByUid(compileResult->Uid, false)) {
766-
QueryCache.EraseByUid(compileResult->Uid);
769+
if (!hasTempTablesNameClashes) {
770+
if (QueryCache.FindByUid(compileResult->Uid, false)) {
771+
QueryCache.EraseByUid(compileResult->Uid);
772+
}
767773
}
768774
}
769775

@@ -814,12 +820,43 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
814820
StartCheckQueriesTtlTimer();
815821
}
816822

823+
bool HasTempTablesNameClashes(
824+
TKqpCompileResult::TConstPtr compileResult,
825+
TKqpTempTablesState::TConstPtr tempTablesState, bool withSessionId = false) {
826+
if (!compileResult) {
827+
return false;
828+
}
829+
if (!compileResult->PreparedQuery) {
830+
return false;
831+
}
832+
833+
return compileResult->PreparedQuery->HasTempTables(tempTablesState, withSessionId);
834+
}
835+
836+
void UpdateQueryCache(TKqpCompileResult::TConstPtr compileResult, bool keepInCache) {
837+
if (QueryCache.FindByUid(compileResult->Uid, false)) {
838+
QueryCache.Replace(compileResult);
839+
} else if (keepInCache) {
840+
if (QueryCache.Insert(compileResult, TableServiceConfig.GetEnableAstCache())) {
841+
Counters->CompileQueryCacheEvicted->Inc();
842+
}
843+
if (compileResult->Query && compileResult->Query->Settings.IsPrepareQuery) {
844+
if (InsertPreparingQuery(compileResult, true)) {
845+
Counters->CompileQueryCacheEvicted->Inc();
846+
};
847+
}
848+
}
849+
}
850+
817851
void Handle(TEvKqp::TEvParseResponse::TPtr& ev, const TActorContext& ctx) {
818852
auto& parseResult = ev->Get()->AstResult;
819853
auto& query = ev->Get()->Query;
820854
auto compileRequest = RequestsQueue.FinishActiveRequest(query);
821855
if (parseResult && parseResult->Ast->IsOk()) {
822856
auto compileResult = QueryCache.FindByAst(query, *parseResult->Ast, compileRequest.KeepInCache);
857+
if (HasTempTablesNameClashes(compileResult, compileRequest.TempTablesState)) {
858+
compileResult = nullptr;
859+
}
823860
if (compileResult) {
824861
Counters->ReportQueryCacheHit(compileRequest.DbCounters, true);
825862

‎ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,7 @@ class TKqpSchemeExecuter : public TActorBootstrapped<TKqpSchemeExecuter> {
113113
}
114114

115115
case NKqpProto::TKqpSchemeOperation::kDropTable: {
116-
auto modifyScheme = schemeOp.GetDropTable();
117-
if (Temporary) {
118-
auto* dropTable = modifyScheme.MutableDrop();
119-
dropTable->SetName(dropTable->GetName() + SessionId);
120-
}
116+
const auto& modifyScheme = schemeOp.GetDropTable();
121117
ev->Record.MutableTransaction()->MutableModifyScheme()->CopyFrom(modifyScheme);
122118
break;
123119
}

‎ydb/core/kqp/gateway/kqp_metadata_loader.cpp‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,16 @@ struct NavigateEntryResult {
3030
std::optional<TString> QueryName;
3131
};
3232

33-
NavigateEntryResult CreateNavigateEntry(const TString& cluster, const TString& path,
34-
const NYql::IKikimrGateway::TLoadTableMetadataSettings& settings, TKqpTempTablesState::TConstPtr tempTablesState = nullptr) {
33+
NavigateEntryResult CreateNavigateEntry(const TString& path,
34+
const NYql::IKikimrGateway::TLoadTableMetadataSettings& settings, TKqpTempTablesState::TConstPtr tempTablesState = nullptr) {
3535
TNavigate::TEntry entry;
3636
TString currentPath = path;
3737
std::optional<TString> queryName = std::nullopt;
3838
if (tempTablesState) {
39-
auto tempTablesIt = tempTablesState->TempTables.find(std::make_pair(cluster, currentPath));
40-
if (tempTablesState->SessionId && tempTablesIt != tempTablesState->TempTables.end()) {
39+
auto tempTablesInfoIt = tempTablesState->FindInfo(currentPath, false);
40+
if (tempTablesInfoIt != tempTablesState->TempTables.end()) {
4141
queryName = currentPath;
42-
currentPath = currentPath + *tempTablesState->SessionId;
42+
currentPath = currentPath + tempTablesState->SessionId;
4343
}
4444
}
4545
entry.Path = SplitPath(currentPath);
@@ -50,10 +50,8 @@ NavigateEntryResult CreateNavigateEntry(const TString& cluster, const TString& p
5050
return {entry, currentPath, queryName};
5151
}
5252

53-
NavigateEntryResult CreateNavigateEntry(const TString& cluster,
54-
const std::pair<TIndexId, TString>& pair,
53+
NavigateEntryResult CreateNavigateEntry(const std::pair<TIndexId, TString>& pair,
5554
const NYql::IKikimrGateway::TLoadTableMetadataSettings& settings, TKqpTempTablesState::TConstPtr tempTablesState = nullptr) {
56-
Y_UNUSED(cluster);
5755
Y_UNUSED(tempTablesState);
5856

5957
TNavigate::TEntry entry;
@@ -701,8 +699,8 @@ NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMeta
701699

702700
const auto externalEntryItem = CreateNavigateExternalEntry(id, settings.WithExternalDatasources_);
703701
Y_ABORT_UNLESS(!settings.WithExternalDatasources_ || externalEntryItem, "External data source must be resolved using path only");
704-
auto resNavigate = settings.WithExternalDatasources_ ? *externalEntryItem : CreateNavigateEntry(cluster,
705-
id, settings, TempTablesState);
702+
auto resNavigate = settings.WithExternalDatasources_ ? *externalEntryItem : CreateNavigateEntry(id,
703+
settings, TempTablesState);
706704
const auto entry = resNavigate.Entry;
707705
const auto queryName = resNavigate.QueryName;
708706
const auto externalEntry = settings.WithExternalDatasources_ ? std::optional<NavigateEntryResult>{} : externalEntryItem;

‎ydb/core/kqp/provider/yql_kikimr_provider.cpp‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -121,11 +121,13 @@ struct TKikimrData {
121121
const TKikimrTableDescription* TKikimrTablesData::EnsureTableExists(const TString& cluster,
122122
const TString& table, TPositionHandle pos, TExprContext& ctx) const
123123
{
124-
auto tempTable = TempTables.FindPtr(table);
125-
126124
auto tablePath = table;
127-
if (tempTable) {
128-
tablePath = *tempTable;
125+
if (TempTablesState) {
126+
auto tempTableInfoIt = TempTablesState->FindInfo(table, true);
127+
128+
if (tempTableInfoIt != TempTablesState->TempTables.end()) {
129+
tablePath = tempTableInfoIt->first;
130+
}
129131
}
130132

131133
auto desc = Tables.FindPtr(std::make_pair(cluster, tablePath));
@@ -141,11 +143,13 @@ const TKikimrTableDescription* TKikimrTablesData::EnsureTableExists(const TStrin
141143
}
142144

143145
TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database, const TString& table, ETableType tableType) {
144-
auto tempTable = TempTables.FindPtr(table);
145-
146146
auto tablePath = table;
147-
if (tempTable) {
148-
tablePath = *tempTable;
147+
if (TempTablesState) {
148+
auto tempTableInfoIt = TempTablesState->FindInfo(table, true);
149+
150+
if (tempTableInfoIt != TempTablesState->TempTables.end()) {
151+
tablePath = tempTableInfoIt->first;
152+
}
149153
}
150154

151155
if (!Tables.FindPtr(std::make_pair(cluster, tablePath))) {
@@ -165,11 +169,13 @@ TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster
165169
}
166170

167171
TKikimrTableDescription& TKikimrTablesData::GetTable(const TString& cluster, const TString& table) {
168-
auto tempTable = TempTables.FindPtr(table);
169-
170172
auto tablePath = table;
171-
if (tempTable) {
172-
tablePath = *tempTable;
173+
if (TempTablesState) {
174+
auto tempTableInfoIt = TempTablesState->FindInfo(table, true);
175+
176+
if (tempTableInfoIt != TempTablesState->TempTables.end()) {
177+
tablePath = tempTableInfoIt->first;
178+
}
173179
}
174180

175181
auto desc = Tables.FindPtr(std::make_pair(cluster, tablePath));
@@ -181,12 +187,13 @@ TKikimrTableDescription& TKikimrTablesData::GetTable(const TString& cluster, con
181187
const TKikimrTableDescription& TKikimrTablesData::ExistingTable(const TStringBuf& cluster,
182188
const TStringBuf& table) const
183189
{
184-
auto tempTable = TempTables.FindPtr(table);
185-
186190
auto tablePath = table;
191+
if (TempTablesState) {
192+
auto tempTableInfoIt = TempTablesState->FindInfo(table, true);
187193

188-
if (tempTable) {
189-
tablePath = *tempTable;
194+
if (tempTableInfoIt != TempTablesState->TempTables.end()) {
195+
tablePath = tempTableInfoIt->first;
196+
}
190197
}
191198

192199
auto desc = Tables.FindPtr(std::make_pair(TString(cluster), TString(tablePath)));

‎ydb/core/kqp/provider/yql_kikimr_provider.h‎

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -200,16 +200,12 @@ class TKikimrTablesData : public TThrRefBase {
200200
}
201201

202202
void SetTempTables(NKikimr::NKqp::TKqpTempTablesState::TConstPtr tempTablesState) {
203-
if (tempTablesState) {
204-
for (const auto& [path, info] : tempTablesState->TempTables) {
205-
TempTables[path.second + *tempTablesState->SessionId] = path.second;
206-
}
207-
}
203+
TempTablesState = std::move(tempTablesState);
208204
}
209205

210206
private:
211207
THashMap<std::pair<TString, TString>, TKikimrTableDescription> Tables;
212-
THashMap<TString, TString> TempTables;
208+
NKikimr::NKqp::TKqpTempTablesState::TConstPtr TempTablesState;
213209
};
214210

215211
enum class TYdbOperation : ui32 {
@@ -288,11 +284,7 @@ class TKikimrTransactionContextBase : public TThrRefBase {
288284
}
289285

290286
void SetTempTables(NKikimr::NKqp::TKqpTempTablesState::TConstPtr tempTablesState) {
291-
if (tempTablesState) {
292-
for (const auto& [path, info] : tempTablesState->TempTables) {
293-
TempTables[path.second] = path.second + *tempTablesState->SessionId;
294-
}
295-
}
287+
TempTablesState = std::move(tempTablesState);
296288
}
297289

298290
template<class IterableKqpTableOps, class IterableKqpTableInfos>
@@ -330,17 +322,16 @@ class TKikimrTransactionContextBase : public TThrRefBase {
330322
}
331323

332324
for (const auto& op : operations) {
333-
const auto& table = [&]() -> const TString& {
334-
const auto tempTable = TempTables.FindPtr(op.GetTable());
335-
if (tempTable) {
336-
return *tempTable;
337-
} else {
338-
return op.GetTable();
339-
}
340-
}();
341-
342325
const auto newOp = TYdbOperation(op.GetOperation());
343326

327+
auto table = op.GetTable();
328+
if (TempTablesState) {
329+
auto tempTableInfoIt = TempTablesState->FindInfo(table, false);
330+
if (tempTableInfoIt != TempTablesState->TempTables.end()) {
331+
table = tempTableInfoIt->first + TempTablesState->SessionId;
332+
}
333+
}
334+
344335
const auto info = tableInfoMap.FindPtr(table);
345336
if (!info) {
346337
TString message = TStringBuilder()
@@ -450,7 +441,7 @@ class TKikimrTransactionContextBase : public TThrRefBase {
450441
THashMap<TString, TYdbOperations> TableOperations;
451442
THashMap<TKikimrPathId, TString> TableByIdMap;
452443
TMaybe<NKikimrKqp::EIsolationLevel> EffectiveIsolationLevel;
453-
THashMap<TString, TString> TempTables;
444+
NKikimr::NKqp::TKqpTempTablesState::TConstPtr TempTablesState;
454445
bool Readonly = false;
455446
bool Invalidated = false;
456447
bool Closed = false;
@@ -535,7 +526,7 @@ class TKikimrSessionContext : public TThrRefBase {
535526
if (TxCtx) {
536527
TxCtx->SetTempTables(tempTablesState);
537528
}
538-
TempTablesState = tempTablesState;
529+
TempTablesState = std::move(tempTablesState);
539530
}
540531

541532
const TIntrusiveConstPtr<NACLib::TUserToken>& GetUserToken() const {

0 commit comments

Comments
 (0)