Skip to content

Commit 5775971

Browse files
lorenzobrunilorenzobruni
committed
[ENHANCEMENT] Improve JSON deserialize performance for large strings #7
[ENHANCEMENT] Allow large objects over 2GB in memory #6 [ENHANCEMENT] Add database selection for Performance monitor #5
1 parent 74ea99a commit 5775971

File tree

19 files changed

+888
-699
lines changed

19 files changed

+888
-699
lines changed

‎rls/API.Library.dll‎

3 KB
Binary file not shown.

‎rls/API.Library.dll.config‎

Lines changed: 210 additions & 202 deletions
Large diffs are not rendered by default.

‎rls/API.Library.pdb‎

2 KB
Binary file not shown.

‎src/API.Library/App.config‎

Lines changed: 210 additions & 202 deletions
Large diffs are not rendered by default.

‎src/API.Library/Entities/API.Map.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public static RESTful_API JSONRPC2RESTful_API(JSONRPC_API JsonRpcApi)
6565
/// <param name="mimeType"></param>
6666
/// <param name="statusCode4NoContent"></param>
6767
/// <returns></returns>
68-
public static RESTful_Output JSONRPC2RESTful_Output(JSONRPC_Output JsonRpcOutput, string mimeType = null, HttpStatusCode statusCode4NoContent = HttpStatusCode.NoContent)
68+
public static RESTful_Output JSONRPC2RESTful_Output(JSONRPC_Output JsonRpcOutput, string mimeType = null, HttpStatusCode statusCode4NoContent = HttpStatusCode.NoContent, HttpStatusCode statusCode4Error = HttpStatusCode.InternalServerError)
6969
{
7070
Log.Instance.Info("Map JSON-RPC Output to RESTful Output");
7171

@@ -78,7 +78,7 @@ public static RESTful_Output JSONRPC2RESTful_Output(JSONRPC_Output JsonRpcOutput
7878
return new RESTful_Output
7979
{
8080
mimeType = null,
81-
statusCode = HttpStatusCode.InternalServerError,
81+
statusCode = statusCode4Error,
8282
response = JsonRpcOutput.error
8383
};
8484
}

‎src/API.Library/Entities/MemCacheD.cs‎

Lines changed: 155 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using Enyim.Caching;
22
using Enyim.Caching.Memcached;
3+
using Newtonsoft.Json.Linq;
34
using System;
45
using System.Collections.Generic;
56
using System.Configuration;
@@ -18,6 +19,11 @@ public static class MemCacheD
1819
/// </summary>
1920
internal static bool API_MEMCACHED_ENABLED = Convert.ToBoolean(ConfigurationManager.AppSettings["API_MEMCACHED_ENABLED"]);
2021

22+
/// <summary>
23+
/// Max size in MB before splitting a string record in sub-cache entries
24+
/// </summary>
25+
internal static uint API_MEMCACHED_MAX_SIZE = Convert.ToUInt32(ConfigurationManager.AppSettings["API_MEMCACHED_MAX_SIZE"]);
26+
2127
/// <summary>
2228
/// Maximum validity in number of seconds that MemCacheD can handle (30 days = 2592000)
2329
/// </summary>
@@ -38,6 +44,11 @@ public static class MemCacheD
3844
/// </summary>
3945
internal static MemcachedClient MemcachedClient = API_MEMCACHED_ENABLED ? new MemcachedClient() : null;
4046

47+
/// <summary>
48+
/// SubKey prefix
49+
/// </summary>
50+
internal static String SubKeyPrefix = "subKey_";
51+
4152
#endregion
4253

4354
#region Methods
@@ -99,7 +110,7 @@ private static string GenerateKey_ADO(string nameSpace, string procedureName, Li
99110
}
100111

101112
/// <summary>
102-
/// Generate the Key for ADO
113+
/// Generate the Key for BSO
103114
/// </summary>
104115
/// <typeparam name="T"></typeparam>
105116
/// <param name="nameSpace"></param>
@@ -142,6 +153,35 @@ private static string GenerateKey_BSO<T>(string nameSpace, string className, str
142153
}
143154
}
144155

156+
/// <summary>
157+
/// Get a SubKey for a Key
158+
/// </summary>
159+
/// <param name="key"></param>
160+
/// <returns></returns>
161+
private static string GetSubKey(string key)
162+
{
163+
return SubKeyPrefix + key;
164+
}
165+
166+
/// <summary>
167+
/// Check if a subKey exists
168+
/// </summary>
169+
/// <param name="key"></param>
170+
/// <returns></returns>
171+
private static bool IsSubKey(dynamic data, string key)
172+
{
173+
// A key must be either a String or a JValue returned from deserialisation
174+
if (data.GetType() == typeof(String) || data.GetType() == typeof(JValue))
175+
{
176+
// Check the explicit casting to String against the subKey
177+
if ((String)data == GetSubKey(key))
178+
{
179+
return true;
180+
}
181+
}
182+
return false;
183+
}
184+
145185
/// <summary>
146186
/// Generate the Hash
147187
/// </summary>
@@ -326,9 +366,33 @@ public static bool Store_BSO<T>(string nameSpace, string className, string metho
326366
/// <param name="key"></param>
327367
/// <param name="value"></param>
328368
/// <param name="validFor"></param>
369+
/// <param name="repository"></param>
329370
/// <returns></returns>
330371
private static bool Store(string key, MemCachedD_Value value, TimeSpan validFor, string repository)
331372
{
373+
// Check if data is of type String or JValue
374+
if (value.data.GetType() == typeof(String) || value.data.GetType() == typeof(JValue))
375+
{
376+
// Cast data to String and check if oversized
377+
string sData = (String)value.data;
378+
if (sData.Length * sizeof(Char) > API_MEMCACHED_MAX_SIZE * 1024 * 1024)
379+
{
380+
// Get a subKey
381+
string subKey = GetSubKey(key);
382+
383+
// SubStore the data by subKey
384+
if (SubStore(subKey, sData, validFor, repository))
385+
{
386+
// Override data with the subKey to fish it out later
387+
value.data = subKey;
388+
}
389+
else
390+
{
391+
return false;
392+
}
393+
}
394+
}
395+
332396
try
333397
{
334398
// The value must be serialised
@@ -371,6 +435,57 @@ private static bool Store(string key, MemCachedD_Value value, TimeSpan validFor,
371435
}
372436
}
373437

438+
/// <summary>
439+
/// SubStore the data string by subKey
440+
/// </summary>
441+
/// <param name="subKey"></param>
442+
/// <param name="data"></param>
443+
/// <param name="validFor"></param>
444+
/// <param name="repository"></param>
445+
/// <returns></returns>
446+
private static bool SubStore(string subKey, string data, TimeSpan validFor, string repository)
447+
{
448+
try
449+
{
450+
// The data is a string, no need to serialize
451+
Log.Instance.Info("SubCache Size Uncompressed (Byte): " + data.Length * sizeof(Char));
452+
453+
// The data must be compressed
454+
string subCacheCompressed = Utility.GZipCompress(data);
455+
Log.Instance.Info("SubCache Size Compressed (Byte): " + subCacheCompressed.Length * sizeof(Char));
456+
457+
bool isStored = false;
458+
459+
// Fix the MemCacheD issue with bad Windows' compiled version: use validFor instead of expiresAt
460+
// validFor and expiresAt match each other
461+
isStored = MemcachedClient.Store(StoreMode.Set, subKey, subCacheCompressed, validFor);
462+
463+
// Store Value by Key
464+
if (isStored)
465+
{
466+
Log.Instance.Info("SubStore succesfull: " + subKey);
467+
468+
// Add the cached record into a Repository
469+
if (!String.IsNullOrEmpty(repository))
470+
{
471+
CasRepositoryStore(subKey, repository);
472+
}
473+
474+
return true;
475+
}
476+
else
477+
{
478+
Log.Instance.Fatal("SubStore failed: " + subKey);
479+
return false;
480+
}
481+
}
482+
catch (Exception e)
483+
{
484+
Log.Instance.Fatal(e);
485+
return false;
486+
}
487+
}
488+
374489
/// <summary>
375490
/// Add a Key into a Cas Repository
376491
/// N.B. Cas records DO NOT exipire, see Enyim inline documentation
@@ -578,12 +693,10 @@ private static MemCachedD_Value Get(string key)
578693
Log.Instance.Info("Cache Valid For (s): " + cacheValidFor.TotalSeconds.ToString());
579694
Log.Instance.Info("Cache Has Data: " + cache.hasData.ToString());
580695

581-
if (!Convert.ToBoolean(cache.hasData))
582-
{
583-
Log.Instance.Info("Force removal of cache without data");
584-
// Remove the record with no data
585-
Remove(key);
586-
}
696+
// Init subKey
697+
string subKey = GetSubKey(key);
698+
// Init isSubKey
699+
bool isSubKey = IsSubKey(cache.data, key);
587700

588701
// double check the cache record is still valid if not cleared by the garbage collector
589702
if (cacheExpiresAt > DateTime.Now
@@ -594,22 +707,45 @@ private static MemCachedD_Value Get(string key)
594707
value.expiresAt = cacheExpiresAt;
595708
value.validFor = cacheValidFor;
596709
value.hasData = Convert.ToBoolean(cache.hasData);
597-
value.data = cache.data;
710+
value.data = isSubKey ? null : cache.data;
711+
712+
// Check for data in the subKey
713+
if (isSubKey)
714+
{
715+
// Get subCacheCompressed from the subKey
716+
string subCacheCompressed = MemcachedClient.Get<string>(subKey);
717+
718+
if (!String.IsNullOrEmpty(subCacheCompressed))
719+
{
720+
Log.Instance.Info("SubCache found: " + subKey);
721+
Log.Instance.Info("SubCache Size Compressed (Byte): " + subCacheCompressed.Length * sizeof(Char));
722+
723+
// Decompress the cache
724+
string subCache = Utility.GZipDecompress(subCacheCompressed);
725+
Log.Instance.Info("SubCache Size Decompressed (Byte): " + subCache.Length * sizeof(Char));
726+
727+
value.data = subCache;
728+
}
729+
else
730+
{
731+
Log.Instance.Info("SubCache not found: " + key);
732+
}
733+
}
598734

599735
return value;
600736
}
601737
else
602738
{
603-
Log.Instance.Info("Force removal of expired cache");
739+
Log.Instance.Info("Forced removal of expired cache");
604740
// Remove the expired record
605741
Remove(key);
742+
606743
}
607744
}
608745
else
609746
{
610747
Log.Instance.Info("Cache not found: " + key);
611748
}
612-
613749
}
614750
catch (Exception e)
615751
{
@@ -666,7 +802,15 @@ private static bool Remove(string key)
666802

667803
try
668804
{
669-
// Remove the record by the Key
805+
string subKey = GetSubKey(key);
806+
807+
// Remove the (optional) subKey
808+
if (MemcachedClient.Remove(subKey))
809+
{
810+
Log.Instance.Info("SubRemoval succesfull: " + subKey);
811+
}
812+
813+
// Remove the Key
670814
if (MemcachedClient.Remove(key))
671815
{
672816
Log.Instance.Info("Removal succesfull: " + key);

‎src/API.Library/Entities/Performance.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ internal static class Performance
1313
/// Flag to indicate if Performance is enabled
1414
/// </summary>
1515
internal static bool API_PERFORMANCE_ENABLED = Convert.ToBoolean(ConfigurationManager.AppSettings["API_PERFORMANCE_ENABLED"]);
16+
internal static string API_PERFORMANCE_DATABASE = ConfigurationManager.AppSettings["API_PERFORMANCE_DATABASE"];
1617

1718
internal static PerformanceCounter ProcessorPercentage = API_PERFORMANCE_ENABLED ? new PerformanceCounter("Processor", "% Processor Time", "_Total") : null;
1819
internal static PerformanceCounter MemoryAvailableMBytes = API_PERFORMANCE_ENABLED ? new PerformanceCounter("Memory", "Available MBytes") : null;
@@ -107,7 +108,7 @@ protected virtual void Dispose(bool disposing)
107108
if (disposing)
108109
{
109110
// Store data
110-
Performance_ADO.Create(new ADO(), items);
111+
Performance_ADO.Create(String.IsNullOrEmpty(Performance.API_PERFORMANCE_DATABASE) ? new ADO() : new ADO(Performance.API_PERFORMANCE_DATABASE), items);
111112
}
112113
}
113114
}

‎src/API.Library/Properties/AssemblyInfo.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@
3131
// You can specify all the values or you can default the Build and Revision Numbers
3232
// by using the '*' as shown below:
3333
// [assembly: AssemblyVersion("1.0.*")]
34-
[assembly: AssemblyVersion("4.4.0")]
35-
[assembly: AssemblyFileVersion("4.4.0")]
34+
[assembly: AssemblyVersion("4.4.1")]
35+
[assembly: AssemblyFileVersion("4.4.1")]
3636

3737
// Configure log4net using the Web.config file by default
3838
[assembly: log4net.Config.XmlConfigurator(Watch = true)]

‎test/API.Test/API.Test.csproj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,9 @@
7272
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
7373
</PropertyGroup>
7474
<ItemGroup>
75-
<Reference Include="API.Library, Version=4.4.0, Culture=neutral, processorArchitecture=MSIL">
75+
<Reference Include="API.Library, Version=4.4.1, Culture=neutral, processorArchitecture=MSIL">
7676
<SpecificVersion>False</SpecificVersion>
77-
<HintPath>..\packages\API.Library.4.4.0\API.Library.dll</HintPath>
77+
<HintPath>..\packages\API.Library.4.4.1\API.Library.dll</HintPath>
7878
</Reference>
7979
<Reference Include="Enyim.Caching, Version=2.16.0.0, Culture=neutral, PublicKeyToken=cec98615db04012e, processorArchitecture=MSIL">
8080
<HintPath>..\packages\EnyimMemcached.2.16.0\lib\net35\Enyim.Caching.dll</HintPath>

‎test/API.Test/Web.Live.config‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@
148148
<!-- ADO - Execution timeout in seconds -->
149149
<add key="API_ADO_EXECUTION_TIMEOUT" value="600" />
150150
<!-- ADO - Bulk Copy timeout in seconds -->
151-
<add key="API_ADO_BULKCOPY_TIMEOUT" value="600" />
151+
<add key="API_ADO_BULKCOPY_TIMEOUT" value="600" />
152152
<!-- ADO - Bulk Copy BatchSize in rows (below 5000 to avoid Table locking) -->
153153
<add key="API_ADO_BULKCOPY_BATCHSIZE" value="4999" />
154154

@@ -159,6 +159,8 @@
159159
-->
160160
<!-- MemCacheD - Switch on [TRUE] or off [FALSE] the MemCacheD -->
161161
<add key="API_MEMCACHED_ENABLED" value="TRUE" />
162+
<!-- MemCacheD - Set the max size in MB before splitting a string record in sub-cache entries -->
163+
<add key="API_MEMCACHED_MAX_SIZE" value="128" />
162164
<!-- MemCacheD - Maximum validity in number of seconds that MemCacheD can handle (30 days = 2592000) -->
163165
<add key="API_MEMCACHED_MAX_VALIDITY" value="2592000" />
164166
<!-- MemCacheD - Salsa code to isolate the cache records form other applications or environments -->
@@ -171,6 +173,8 @@
171173
-->
172174
<!-- Performance - Switch on [TRUE] or off [FALSE] the Performance -->
173175
<add key="API_PERFORMANCE_ENABLED" value="FALSE" />
176+
<!-- Performance - Choose the Database connection string where to store the records -->
177+
<add key="API_PERFORMANCE_DATABASE" value="defaultConnection" />
174178

175179
</appSettings>
176180

0 commit comments

Comments
 (0)