implement distributed meta cache, optimize database list

ydb-platform · May 21, 2024 · c287ee8 · c287ee8
1 parent ffff4a0
commit c287ee8
Show file tree

Hide file tree

Showing 15 changed files with 334 additions and 205 deletions.
diff --git a/ydb/core/viewer/json_tenantinfo.h b/ydb/core/viewer/json_tenantinfo.h
@@ -47,6 +47,7 @@ class TJsonTenantInfo : public TViewerPipeClient<TJsonTenantInfo> {
     bool SystemTablets = false;
     bool Storage = false;
     bool Nodes = false;
+    bool Users = false;
     bool OffloadMerge = false;
     THashMap<TString, std::vector<TNodeId>> TenantNodes;
     THashMap<TString, NKikimrViewer::TEvViewerResponse> OffloadMergedTabletStateResponse;
@@ -103,6 +104,7 @@ class TJsonTenantInfo : public TViewerPipeClient<TJsonTenantInfo> {
         SystemTablets = FromStringWithDefault<bool>(params.Get("system_tablets"), Tablets); // Tablets here is by design
         Storage = FromStringWithDefault<bool>(params.Get("storage"), Storage);
         Nodes = FromStringWithDefault<bool>(params.Get("nodes"), Nodes);
+        Users = FromStringWithDefault<bool>(params.Get("users"), Users);
         User = params.Get("user");
         Path = params.Get("path");
         OffloadMerge = FromStringWithDefault<bool>(params.Get("offload_merge"), OffloadMerge);
@@ -521,16 +523,18 @@ class TJsonTenantInfo : public TViewerPipeClient<TJsonTenantInfo> {
                     continue;
                 }
                 std::unordered_set<TString> users;
-                if (entry.SecurityObject) {
-                    users.emplace(entry.SecurityObject->GetOwnerSID());
-                    for (const NACLibProto::TACE& ace : entry.SecurityObject->GetACL().GetACE()) {
-                        if (ace.GetAccessType() == (ui32)NACLib::EAccessType::Allow) {
-                            users.emplace(ace.GetSID());
+                if(!User.empty() || Users) {
+                    if (entry.SecurityObject) {
+                        users.emplace(entry.SecurityObject->GetOwnerSID());
+                        for (const NACLibProto::TACE& ace : entry.SecurityObject->GetACL().GetACE()) {
+                            if (ace.GetAccessType() == (ui32)NACLib::EAccessType::Allow) {
+                                users.emplace(ace.GetSID());
+                            }
                         }
                     }
-                }
-                if (!IsValidOwner(users)) {
-                    continue;
+                    if (!IsValidOwner(users)) {
+                        continue;
+                    }
                 }
                 NKikimrViewer::TTenant& tenant = *Result.AddTenantInfo();
                 auto itTenantByPath = TenantByPath.find(path);

diff --git a/ydb/library/actors/http/http_proxy_outgoing.cpp b/ydb/library/actors/http/http_proxy_outgoing.cpp
@@ -250,7 +250,6 @@ class TOutgoingConnectionActor : public NActors::TActor<TOutgoingConnectionActor
         if (event->Get()->Timeout) {
             ConnectionTimeout = event->Get()->Timeout;
         }
-        ALOG_DEBUG(HttpLog, "Scheduling timeout for " << ctx.SelfID << " in " << ConnectionTimeout);
         ctx.Schedule(ConnectionTimeout, new NActors::TEvents::TEvWakeup());
         LastActivity = ctx.Now();
         TBase::Become(&TOutgoingConnectionActor::StateResolving);
@@ -280,12 +279,10 @@ class TOutgoingConnectionActor : public NActors::TActor<TOutgoingConnectionActor
     }
 
     void HandleTimeout(const NActors::TActorContext& ctx) {
-        ALOG_DEBUG(HttpLog, GetSocketName() << "timeout");
         TDuration inactivityTime = ctx.Now() - LastActivity;
         if (inactivityTime >= ConnectionTimeout) {
             FailConnection(ctx, "Connection timed out");
         } else {
-            ALOG_DEBUG(HttpLog, GetSocketName() << "waiting for more");
             ctx.Schedule(Min(ConnectionTimeout - inactivityTime, TDuration::MilliSeconds(100)), new NActors::TEvents::TEvWakeup());
         }
     }

diff --git a/ydb/mvp/core/core_ydb.cpp b/ydb/mvp/core/core_ydb.cpp
@@ -11,6 +11,42 @@ NJson::TJsonWriterConfig GetDefaultJsonWriterConfig() {
 
 NJson::TJsonReaderConfig NMVP::THandlerActorYdb::JsonReaderConfig;
 NJson::TJsonWriterConfig NMVP::THandlerActorYdb::JsonWriterConfig = GetDefaultJsonWriterConfig();
+TMap<std::pair<TStringBuf, TStringBuf>, TYdbUnitResources> DefaultUnitResources =
+    {
+        {
+            {
+                "compute",
+                "slot"
+            },
+            {
+                10.0,
+                (ui64)50*1024*1024*1024,
+                0
+            }
+        },
+        {
+            {
+                "storage",
+                "hdd"
+            },
+            {
+                0,
+                0,
+                (ui64)500*1024*1024*1024
+            }
+        },
+        {
+            {
+                "storage",
+                "ssd"
+            },
+            {
+                0,
+                0,
+                (ui64)100*1024*1024*1024
+            }
+        }
+    };
 TString TYdbLocation::UserToken;
 TString TYdbLocation::CaCertificate;
 TString TYdbLocation::SslCertificate;

diff --git a/ydb/mvp/core/core_ydb.h b/ydb/mvp/core/core_ydb.h
@@ -140,15 +140,17 @@ struct TYdbUnitResources {
     }
 };
 
+extern TMap<std::pair<TStringBuf, TStringBuf>, TYdbUnitResources> DefaultUnitResources;
+
 struct TYdbLocation {
     TString Name;
     TString Environment;
-    TVector<std::pair<TStringBuf, TStringBuf>> Endpoints;
-    const TString RootDomain;
+    TVector<std::pair<TString, TString>> Endpoints;
+    TString RootDomain;
     TVector<TStringBuf> DataCenters;
     const TMap<std::pair<TStringBuf, TStringBuf>, TYdbUnitResources>& UnitResources;
     ui32 NotificationsEnvironmentId;
-    bool Disabled;
+    bool Disabled = false;
     TAtomicSingleton<NYdb::TDriver> Driver;
     TAtomicSingleton<NYdbGrpc::TGRpcClientLow> GRpcClientLow;
     static TString UserToken;
@@ -159,19 +161,29 @@ struct TYdbLocation {
 
     TYdbLocation(const TString& name,
                  const TString& environment,
-                 const TVector<std::pair<TStringBuf, TStringBuf>>& endpoints,
+                 const TVector<std::pair<TString, TString>>& endpoints,
+                 const TString& rootDomain)
+        : Name(name)
+        , Environment(environment)
+        , Endpoints(endpoints)
+        , RootDomain(rootDomain)
+        , UnitResources(DefaultUnitResources)
+    {}
+
+    TYdbLocation(const TString& name,
+                 const TString& environment,
+                 const TVector<std::pair<TString, TString>>& endpoints,
                  const TString& rootDomain,
                  const TVector<TStringBuf>& dataCenters,
                  const TMap<std::pair<TStringBuf, TStringBuf>, TYdbUnitResources>& unitResources,
-                 ui32 notificationsEnvironmendId = 0)
+                 ui32 notificationsEnvironmentId = 0)
         : Name(name)
         , Environment(environment)
         , Endpoints(endpoints)
         , RootDomain(rootDomain)
         , DataCenters(dataCenters)
         , UnitResources(unitResources)
-        , NotificationsEnvironmentId(notificationsEnvironmendId)
-        , Disabled(false)
+        , NotificationsEnvironmentId(notificationsEnvironmentId)
     {}
 
     static TString GetUserToken() {

diff --git a/ydb/mvp/core/mvp_log.h b/ydb/mvp/core/mvp_log.h
@@ -23,5 +23,4 @@ enum EService : NActors::NLog::EComponent {
 #define BLOG_GRPC_DC(context, stream) LOG_DEBUG_S(context, EService::GRPC, stream)
 #define BLOG_QUERY_I(stream) LOG_INFO_S(*NActors::TlsActivationContext, EService::QUERY, stream)
 
-
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -23,5 +23,4 @@ enum EService : NActors::NLog::EComponent {
		#define BLOG_GRPC_DC(context, stream) LOG_DEBUG_S(context, EService::GRPC, stream)
		#define BLOG_QUERY_I(stream) LOG_INFO_S(*NActors::TlsActivationContext, EService::QUERY, stream)


		}