Cache bigmap ptrs

Tzkt.Api/Repositories/BigMapsRepository.cs

@@ -15,11 +15,13 @@ namespace Tzkt.Api.Repositories
     public class BigMapsRepository : DbConnection
     {
         readonly AccountsCache Accounts;
+        readonly BigMapsCache BigMaps;
         readonly TimeCache Times;
 
-        public BigMapsRepository(AccountsCache accounts, TimeCache times, IConfiguration config) : base(config)
+        public BigMapsRepository(AccountsCache accounts, BigMapsCache bigMaps, TimeCache times, IConfiguration config) : base(config)
         {
             Accounts = accounts;
+            BigMaps = bigMaps;
             Times = times;
         }
 
@@ -74,6 +76,7 @@ public async Task<BigMap> Get(int contractId, string path, MichelineFormat miche
                 SELECT  *
                 FROM    ""BigMaps""
                 WHERE   ""ContractId"" = @id
+                AND 	""Active"" = true
                 AND     ""StoragePath"" LIKE @path";
 
             using var db = GetConnection();
@@ -84,20 +87,9 @@ public async Task<BigMap> Get(int contractId, string path, MichelineFormat miche
             return ReadBigMap(row ?? rows.FirstOrDefault(), micheline);
         }
 
-        public async Task<int?> GetPtr(int contractId, string path)
+        public Task<int?> GetPtr(int contractId, string path)
         {
-            var sql = @"
-                SELECT  ""Ptr"", ""StoragePath""
-                FROM    ""BigMaps""
-                WHERE   ""ContractId"" = @id
-                AND     ""StoragePath"" LIKE @path";
-
-            using var db = GetConnection();
-            var rows = await db.QueryAsync(sql, new { id = contractId, path = $"%{path}" });
-            if (!rows.Any()) return null;
-
-            var row = rows.FirstOrDefault(x => x.StoragePath == path);
-            return (row ?? rows.FirstOrDefault())?.Ptr;
+            return BigMaps.GetPtrAsync(contractId, path);
         }
 
         public async Task<IEnumerable<BigMap>> Get(

Tzkt.Api/Services/Cache/BigMaps/BigMapsCache.cs

@@ -0,0 +1,128 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using Dapper;
+
+namespace Tzkt.Api.Services.Cache
+{
+    public class BigMapsCache : DbConnection
+    {
+        const int MaxPtrs = 4096;
+        readonly Dictionary<string, RawBigMap> Cached = new(MaxPtrs);
+        readonly SemaphoreSlim Sema = new(1);
+        int LastUpdate;
+
+        readonly StateCache State;
+        readonly ILogger Logger;
+
+        public BigMapsCache(StateCache state, IConfiguration config, ILogger<BigMapsCache> logger) : base(config)
+        {
+            State = state;
+            Logger = logger;
+            LastUpdate = state.Current.Level;
+        }
+
+        public async Task UpdateAsync()
+        {
+            Logger.LogDebug("Updating bigmaps cache...");
+            var from = Math.Min(LastUpdate, State.ValidLevel);
+            try
+            {
+                await Sema.WaitAsync();
+                #region check reorg
+                if (State.Reorganized)
+                {
+                    List<string> corrupted;
+                    corrupted = Cached
+                        .Where(x => x.Value.LastLevel > from)
+                        .Select(x => x.Key)
+                        .ToList();
+
+                    foreach (var key in corrupted)
+                        Cached.Remove(key);
+
+                    Logger.LogDebug("Removed {0} corrupted BigMaps", corrupted.Count);
+                }
+                #endregion
+
+                var sql = @"
+                    SELECT  ""Ptr"", ""LastLevel"", ""Active""
+                    FROM    ""BigMaps""
+                    WHERE   ""LastLevel"" > @from";
+
+                using var db = GetConnection();
+                var rows = await db.QueryAsync(sql, new { from });
+                if (!rows.Any()) return;
+
+                var removed = rows.Where(x => !x.Active).Select(x => (int)x.Ptr).ToHashSet();
+                var keys = Cached.Where(x => removed.Contains(x.Value.Ptr)).Select(x => x.Key).ToList();
+                foreach (var key in keys)
+                    Cached.Remove(key);
+
+                var updated = rows.Where(x => x.Active).ToDictionary(x => (int)x.Ptr, x => (int)x.LastLevel);
+                foreach (var item in Cached.Values)
+                    if (updated.TryGetValue(item.Ptr, out var lastLevel))
+                        item.LastLevel = lastLevel;
+
+                LastUpdate = State.Current.Level;
+                Logger.LogDebug("Updated {0} bigmaps since block {1}", removed.Count + updated.Count, from);
+            }
+            finally
+            {
+                Sema.Release();
+            }
+        }
+
+        public async Task<int?> GetPtrAsync(int contractId, string path)
+        {
+            var key = $"{contractId}~{path}";
+            if (!Cached.TryGetValue(key, out var item))
+            {
+                try
+                {
+                    await Sema.WaitAsync();
+                    if (!Cached.TryGetValue(key, out item))
+                    {
+                        var sql = @"
+                        SELECT  ""Ptr"", ""StoragePath"", ""LastLevel""
+                        FROM    ""BigMaps""
+                        WHERE   ""ContractId"" = @id
+                        AND 	""Active"" = true
+                        AND     ""StoragePath"" LIKE @name";
+
+                        using var db = GetConnection();
+                        var rows = await db.QueryAsync(sql, new { id = contractId, name = $"%{path}" });
+                        if (!rows.Any()) return null;
+
+                        var row = rows.FirstOrDefault(x => x.StoragePath == path) ?? rows.First();
+                        item = new RawBigMap
+                        {
+                            Ptr = row.Ptr,
+                            LastLevel = row.LastLevel
+                        };
+                        CheckSpace();
+                        Cached.Add(key, item);
+                    }
+                }
+                finally
+                {
+                    Sema.Release();
+                }
+            }
+            return item.Ptr;
+        }
+
+        void CheckSpace()
+        {
+            if (Cached.Count >= MaxPtrs)
+            {
+                foreach (var key in Cached.Keys.Take(MaxPtrs / 4).ToList())
+                    Cached.Remove(key);
+            }
+        }
+    }
+}

Tzkt.Api/Services/Cache/BigMaps/Models/RawBigMap.cs

@@ -0,0 +1,8 @@
+namespace Tzkt.Api.Services.Cache
+{
+    class RawBigMap
+    {
+        public int Ptr { get; set; }
+        public int LastLevel { get; set; }
+    }
+}

Tzkt.Api/Services/Sync/StateListener.cs

@@ -20,6 +20,7 @@ public class StateListener : BackgroundService
         readonly string ConnectionString;
 
         readonly StateCache State;
+        readonly BigMapsCache BigMaps;
         readonly AccountsCache Accounts;
         readonly ProtocolsCache Protocols;
         readonly QuotesCache Quotes;
@@ -33,6 +34,7 @@ public class StateListener : BackgroundService
 
         public StateListener(
             StateCache state,
+            BigMapsCache bigMaps,
             AccountsCache accounts,
             ProtocolsCache protocols,
             QuotesCache quotes,
@@ -45,6 +47,7 @@ public StateListener(
             ConnectionString = config.GetConnectionString("DefaultConnection");
 
             State = state;
+            BigMaps = bigMaps;
             Accounts = accounts;
             Protocols = protocols;
             Quotes = quotes;
@@ -164,6 +167,7 @@ private async Task NotifyAsync()
 
                 #region update cache
                 await Accounts.UpdateAsync();
+                await BigMaps.UpdateAsync();
                 await Protocols.UpdateAsync();
                 await Quotes.UpdateAsync();
                 await Times.UpdateAsync();

Tzkt.Api/Startup.cs

@@ -35,6 +35,7 @@ public void ConfigureServices(IServiceCollection services)
                 options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection")));
 
             services.AddSingleton<AccountsCache>();
+            services.AddSingleton<BigMapsCache>();
             services.AddSingleton<AliasesCache>();
             services.AddSingleton<ProtocolsCache>();
             services.AddSingleton<QuotesCache>();