Skip to content

Commit f3f4bc9

Browse files
lanesCSOlanesCSO
committed
7.0.16
- [ENHANCEMENT] updated Microsoft.Extensions.DependencyInjection to 9.0.0 - [ENHANCEMENT] updated System.Diagnostics.PerformanceCounter to 9.0.0 - [ENHANCEMENT] updated System.DirectoryServices.AccountManagement to 9.0.0 - [ENHANCEMENT] updated Microsoft.Extensions.Logging to 9.0.0 - [ENHANCEMENT] change parse request from fatal logs to error logs - [ENHANCEMENT] change MemCacheD ValidateExpiry to log errors if changing cache expiry times rather than info - [ENHANCEMENT] created 3 new appsettings properties > int API_CACHE_LOCK_POLL_INTERVAL { get; } > string API_CACHE_LOCK_PREFIX { get; } > int API_CACHE_LOCK_MAX_TIME { get;} - [ENHANCEMENT] added GetSHA256 (copy of function in utility.cs) function to memcached class - [ENHANCEMENT] created custom configuration exception - [ENHANCEMENT] created private function metaCacheLock - [ENHANCEMENT] create new Get_BSO_WITHLOCK functionality for cache stampede protection - [ENHANCEMENT] create new Store_BSO_REMOVELOCK functionality for cache stampede protection - [ENHANCEMENT] added cache tracing for cache lock - [ENHANCEMENT] updated api cache trace table - [ENHANCEMENT] added 2 new values to MemCachedD_Value class > decimal cacheLockDuration > bool cacheLockUsed - [ENHANCEMENT] validated config for memcache on application pool startup - [ENHANCEMENT] added new cache_lock_settings block to cachesettings block of appsettings.json with new keys "CacheSettings": { "API_MEMCACHED_SALSA": "apd_test1_3", "API_MEMCACHED_MAX_VALIDITY": "2592000", "API_MEMCACHED_MAX_SIZE": "128", "API_MEMCACHED_ENABLED": true, "API_CACHE_TRACE_ENABLED": true, "API_CACHE_LOCK_POLL_INTERVAL": "1", "API_CACHE_LOCK_PREFIX1": "LCK", "API_CACHE_LOCK_MAX_TIME": "10", "API_CACHE_LOCK_SETTINGS": { "API_CACHE_LOCK_ENABLED": false, "API_CACHE_LOCK_POLL_INTERVAL": "1", "API_CACHE_LOCK_PREFIX": "LCK", "API_CACHE_LOCK_MAX_TIME": "30" } }, - [ENHANCEMENT] REMOVED public static CacheSettings CacheSettings; from apiserviceshelper
1 parent 96455e0 commit f3f4bc9

File tree

494 files changed

+425439
-116
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

494 files changed

+425439
-116
lines changed

‎db/TD_API_CACHE_TRACE.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
[CCH_SUCCESS] bit,
99
[CCH_COMPRESSED_SIZE] int,
1010
[CCH_EXPIRES_AT] datetime,
11+
[CCH_CACHE_LOCK_USED] BIT NULL,
12+
[CCH_CACHE_LOCK_DURATION] DECIMAL(10, 3) NULL,
1113
PRIMARY KEY CLUSTERED ([CCH_ID] ASC),
1214
)
1315
GO

‎rls/packages/API.Library.7.0.16.nupkg‎

73.1 KB
Binary file not shown.

‎rls/packages/API.Library.7.0.16.symbols.nupkg‎

109 KB
Binary file not shown.

‎src/API.Library/API.Library.csproj‎

Lines changed: 51 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,50 @@
1212
<PackageId>API.Library</PackageId>
1313
<Product>API Library</Product>
1414
<Copyright>Central Statistics Office, Ireland</Copyright>
15-
<Version>7.0.15</Version>
15+
<Version>7.0.16</Version>
1616
<Authors>Central Statistics Office, Ireland</Authors>
1717
<SignAssembly>False</SignAssembly>
1818
<RepositoryUrl>https://github.com/CSOIreland/Server-API-Library</RepositoryUrl>
1919
<PackageReleaseNotes>
20-
- [ENHANCEMENT] added argon2 sha256 methods -- GetArgon2SHA256 and VerifyArgon2SHA256
21-
- [BUG FIX] moved where trace.TrcRequestVerb = requestMethod; as if blockedurls used then verb wouldn't be set
22-
- [BUG FIX] updated GatherTraceInformation to take in an IRequest interface and check the interface type to determine the parameters
23-
- [ENHANCEMENT] upgraded EnyimMemcachedCore from 3.2.1 to 3.2.3
24-
- [ENHANCEMENT] upgraded Microsoft.Extensions.DependencyInjection from 8.0.0 to 8.0.1
25-
- [ENHANCEMENT] upgraded Microsoft.Extensions.Logging from 8.0.0 to 8.0.1
26-
- [ENHANCEMENT] upgraded System.Diagnostics.PerformanceCounter from 8.0.0 to 8.0.1
27-
- [ENHANCEMENT] upgraded System.DirectoryServices.AccountManagement from 8.0.0 to 8.0.1
20+
- [ENHANCEMENT] updated Microsoft.Extensions.DependencyInjection to 9.0.0
21+
- [ENHANCEMENT] updated System.Diagnostics.PerformanceCounter to 9.0.0
22+
- [ENHANCEMENT] updated System.DirectoryServices.AccountManagement to 9.0.0
23+
- [ENHANCEMENT] updated Microsoft.Extensions.Logging to 9.0.0
24+
- [ENHANCEMENT] change parse request from fatal logs to error logs
25+
- [ENHANCEMENT] change MemCacheD ValidateExpiry to log errors if changing cache expiry times rather than info
26+
- [ENHANCEMENT] created 3 new appsettings properties
27+
&gt; int API_CACHE_LOCK_POLL_INTERVAL { get; }
28+
&gt; string API_CACHE_LOCK_PREFIX { get; }
29+
&gt; int API_CACHE_LOCK_MAX_TIME { get;}
30+
- [ENHANCEMENT] added GetSHA256 (copy of function in utility.cs) function to memcached class
31+
- [ENHANCEMENT] created custom configuration exception
32+
- [ENHANCEMENT] created private function metaCacheLock
33+
- [ENHANCEMENT] create new Get_BSO_WITHLOCK functionality for cache stampede protection
34+
- [ENHANCEMENT] create new Store_BSO_REMOVELOCK functionality for cache stampede protection
35+
- [ENHANCEMENT] added cache tracing for cache lock
36+
- [ENHANCEMENT] updated api cache trace table
37+
- [ENHANCEMENT] added 2 new values to MemCachedD_Value class
38+
&gt; decimal cacheLockDuration
39+
&gt; bool cacheLockUsed
40+
- [ENHANCEMENT] validated config for memcache on application pool startup
41+
- [ENHANCEMENT] added new cache_lock_settings block to cachesettings block of appsettings.json with new keys
42+
"CacheSettings": {
43+
"API_MEMCACHED_SALSA": "apd_test1_3",
44+
"API_MEMCACHED_MAX_VALIDITY": "2592000",
45+
"API_MEMCACHED_MAX_SIZE": "128",
46+
"API_MEMCACHED_ENABLED": true,
47+
"API_CACHE_TRACE_ENABLED": true,
48+
"API_CACHE_LOCK_POLL_INTERVAL": "1",
49+
"API_CACHE_LOCK_PREFIX1": "LCK",
50+
"API_CACHE_LOCK_MAX_TIME": "10",
51+
"API_CACHE_LOCK_SETTINGS": {
52+
"API_CACHE_LOCK_ENABLED": false,
53+
"API_CACHE_LOCK_POLL_INTERVAL": "1",
54+
"API_CACHE_LOCK_PREFIX": "LCK",
55+
"API_CACHE_LOCK_MAX_TIME": "30"
56+
}
57+
},
58+
- [ENHANCEMENT] REMOVED public static CacheSettings CacheSettings; from apiserviceshelper
2859
</PackageReleaseNotes>
2960
<RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
3061
<RestoreLockedMode>true</RestoreLockedMode>
@@ -47,16 +78,16 @@
4778
</ItemGroup>
4879

4980
<ItemGroup>
50-
<PackageReference Include="EnyimMemcachedCore" Version="3.2.3" />
81+
<PackageReference Include="EnyimMemcachedCore" Version="3.2.4" />
5182
<PackageReference Include="Konscious.Security.Cryptography.Argon2" Version="1.3.1" />
5283
<PackageReference Include="log4net" Version="2.0.17" />
5384
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
54-
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
55-
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.1" />
85+
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.0" />
86+
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
5687
<PackageReference Include="Microsoft.Extensions.Logging.Log4Net.AspNetCore" Version="8.0.0" />
5788
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
58-
<PackageReference Include="System.Diagnostics.PerformanceCounter" Version="8.0.1" />
59-
<PackageReference Include="System.DirectoryServices.AccountManagement" Version="8.0.1" />
89+
<PackageReference Include="System.Diagnostics.PerformanceCounter" Version="9.0.0" />
90+
<PackageReference Include="System.DirectoryServices.AccountManagement" Version="9.0.0" />
6091
</ItemGroup>
6192

6293

@@ -86,4 +117,9 @@
86117
</PropertyGroup>
87118

88119

89-
</Project>
120+
<PropertyGroup>
121+
<Nullable>enable</Nullable>
122+
</PropertyGroup>
123+
124+
125+
</Project>

‎src/API.Library/Config/CacheConfig.cs‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,38 @@ public bool API_CACHE_TRACE_ENABLED
4949
return _CacheSettingsDelegate.CurrentValue.API_CACHE_TRACE_ENABLED;
5050
}
5151
}
52-
52+
53+
public bool API_CACHE_LOCK_ENABLED
54+
{
55+
get
56+
{
57+
return _CacheSettingsDelegate.CurrentValue.API_CACHE_LOCK_SETTINGS.API_CACHE_LOCK_ENABLED;
58+
}
59+
}
60+
61+
public int API_CACHE_LOCK_POLL_INTERVAL
62+
{
63+
get
64+
{
65+
return _CacheSettingsDelegate.CurrentValue.API_CACHE_LOCK_SETTINGS.API_CACHE_LOCK_POLL_INTERVAL;
66+
}
67+
}
68+
69+
public string API_CACHE_LOCK_PREFIX
70+
{
71+
get
72+
{
73+
return _CacheSettingsDelegate.CurrentValue.API_CACHE_LOCK_SETTINGS.API_CACHE_LOCK_PREFIX;
74+
}
75+
}
76+
77+
public int API_CACHE_LOCK_MAX_TIME
78+
{
79+
get
80+
{
81+
return _CacheSettingsDelegate.CurrentValue.API_CACHE_LOCK_SETTINGS.API_CACHE_LOCK_MAX_TIME;
82+
}
83+
}
84+
5385
}
5486
}

‎src/API.Library/Config/ConfigSettings.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,21 @@ public class CacheSettings
5959
public uint API_MEMCACHED_MAX_SIZE { get; set; }
6060
public bool API_MEMCACHED_ENABLED { get; set; }
6161
public bool API_CACHE_TRACE_ENABLED { get; set; }
62+
public CacheLockSettings API_CACHE_LOCK_SETTINGS { get; set; }
6263
}
6364

65+
66+
public class CacheLockSettings
67+
{
68+
public bool API_CACHE_LOCK_ENABLED { get; set; }
69+
70+
public int API_CACHE_LOCK_POLL_INTERVAL { get; set; }
71+
public string API_CACHE_LOCK_PREFIX { get; set; }
72+
public int API_CACHE_LOCK_MAX_TIME { get; set; }
73+
74+
}
75+
76+
6477
public class APIPerformanceSettings
6578
{
6679
public bool API_PERFORMANCE_ENABLED { get; set; }

‎src/API.Library/Config/ConfigValidation.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
namespace API;
2+
public static class ConfigValidation
3+
{
4+
public static bool cacheSalsaValidation(CacheSettings config)
5+
{
6+
if (config.API_MEMCACHED_ENABLED)
7+
{
8+
if (string.IsNullOrEmpty(config.API_MEMCACHED_SALSA))
9+
{
10+
throw new ConfigurationException("Memcache salsa must not be null");
11+
}
12+
}
13+
return true;
14+
}
15+
16+
public static bool cacheLockValidation(CacheSettings config)
17+
{
18+
if (config.API_CACHE_LOCK_SETTINGS.API_CACHE_LOCK_ENABLED)
19+
{
20+
if (string.IsNullOrEmpty(config.API_CACHE_LOCK_SETTINGS.API_CACHE_LOCK_PREFIX))
21+
{
22+
throw new ConfigurationException("Cache lock prefix must not be null");
23+
}
24+
if (config.API_CACHE_LOCK_SETTINGS.API_CACHE_LOCK_MAX_TIME <= 0)
25+
{
26+
throw new ConfigurationException("Cache lock max time must be greater than 0");
27+
}
28+
if (config.API_CACHE_LOCK_SETTINGS.API_CACHE_LOCK_POLL_INTERVAL <= 0)
29+
{
30+
throw new ConfigurationException("Cache lock poll interval must be greater than 0");
31+
}
32+
}
33+
return true;
34+
}
35+
}
36+

‎src/API.Library/Console/ConsoleConfiguration.cs‎

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Enyim.Caching;
2+
using Enyim.Caching.Memcached;
23
using log4net.Config;
34
using Microsoft.AspNetCore.Builder;
45
using Microsoft.AspNetCore.HttpOverrides;
@@ -20,6 +21,8 @@ public static IHost AddApiLibrary(IConfiguration configuration)
2021
//log name of the machine for identification purposes
2122
log4net.GlobalContext.Properties["MachineName"] = System.Environment.MachineName;
2223

24+
bool memcachedFailedtoLoad = false;
25+
2326

2427
var builder = Host.CreateDefaultBuilder()
2528
.ConfigureAppConfiguration(builder =>
@@ -28,7 +31,22 @@ public static IHost AddApiLibrary(IConfiguration configuration)
2831
builder.AddConfiguration(configuration);
2932
}).ConfigureServices(ser =>
3033
{
31-
ser.AddEnyimMemcached();
34+
35+
try
36+
{
37+
ser.AddEnyimMemcached();
38+
}
39+
catch (Exception ex)
40+
{
41+
Log.Instance.Fatal(ex);
42+
Log.Instance.Fatal("Memcache failed to load");
43+
memcachedFailedtoLoad = true;
44+
}
45+
46+
47+
48+
49+
3250
ser.AddSingleton<ICacheConfig, CacheConfig>();
3351

3452
ser.AddSingleton<IApiConfiguration, ApiConfiguration>();
@@ -51,7 +69,23 @@ public static IHost AddApiLibrary(IConfiguration configuration)
5169
options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto;
5270
});
5371
ser.AddHttpContextAccessor();
54-
ser.Configure<CacheSettings>(configuration.GetSection("CacheSettings"));
72+
73+
ser.Configure<CacheSettings>(configuration.GetSection("CacheSettings"))
74+
.AddOptions<CacheSettings>()
75+
.ValidateDataAnnotations()
76+
.Validate(config =>
77+
{
78+
if (config.API_MEMCACHED_ENABLED)
79+
{
80+
if (!ConfigValidation.cacheSalsaValidation(config))
81+
{
82+
return false;
83+
}
84+
return ConfigValidation.cacheLockValidation(config);
85+
}
86+
return true;
87+
});
88+
5589
ser.Configure<ADOSettings>(configuration.GetSection("ADOSettings"));
5690
ser.Configure<APIConfig>(configuration.GetSection("API_Config"));
5791
ser.Configure<APPConfig>(configuration.GetSection("APP_Config"));
@@ -82,15 +116,34 @@ public static IHost AddApiLibrary(IConfiguration configuration)
82116
ApiServicesHelper.BlockedRequests = BlockedRequests.Value;
83117
ApiServicesHelper.APIConfig = APIConfig.Value;
84118
ApiServicesHelper.APPConfig = APPConfig.Value;
85-
ApiServicesHelper.CacheSettings = CacheSettings.Value;
119+
// ApiServicesHelper.CacheSettings = CacheSettings.Value;
86120

87121
ApiServicesHelper.APIPerformanceSettings = builder.Services.GetRequiredService<IAPIPerformanceConfiguration>();
88122
ApiServicesHelper.DatabaseTracingConfiguration = builder.Services.GetRequiredService<IDatabaseTracingConfiguration>();
89123

90124
//setup memcache here
91-
ApiServicesHelper.CacheConfig = builder.Services.GetRequiredService<ICacheConfig>();
92-
ApiServicesHelper.MemcachedClient = builder.Services.GetRequiredService<IMemcachedClient>();
93-
ApiServicesHelper.CacheD = builder.Services.GetRequiredService<ICacheD>();
125+
try
126+
{
127+
ApiServicesHelper.CacheConfig = builder.Services.GetRequiredService<ICacheConfig>();
128+
ApiServicesHelper.MemcachedClient = builder.Services.GetRequiredService<IMemcachedClient>();
129+
ApiServicesHelper.CacheD = builder.Services.GetRequiredService<ICacheD>();
130+
ServerStats stats = ApiServicesHelper.CacheD.GetStats();
131+
if (stats == null)
132+
{
133+
throw new Exception("Error reading memcache stats");
134+
}
135+
}
136+
catch (ConfigurationException ex)
137+
{
138+
Log.Instance.Fatal(ex);
139+
throw;
140+
}
141+
catch (Exception ex)
142+
{
143+
Log.Instance.Fatal(ex);
144+
Log.Instance.Fatal("Memcache failed to load");
145+
memcachedFailedtoLoad = true;
146+
}
94147

95148

96149
//we need to load API config here as needed for application to work.

‎src/API.Library/Entities/API.JSONRPC.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -287,8 +287,8 @@ private async Task<JSONRPC_Request> ParseRequest(HttpContext httpContext,Cancell
287287
// Deserialize JSON to an Object dynamically
288288
JSONRPC_Request = Utility.JsonDeserialize_IgnoreLoopingReference<JSONRPC_Request>(request);
289289
} catch (Exception e){
290-
Log.Instance.Fatal(request);
291-
Log.Instance.Fatal(e);
290+
Log.Instance.Error(request);
291+
Log.Instance.Error(e);
292292

293293
var error = new JSONRPC_Error { code = -32700 };
294294
await ParseError(httpContext, null, error, sourceToken,trace);

‎src/API.Library/Entities/API.RESTful.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,8 +286,8 @@ Value of HttpContext.Current.Request.Url.PathAndQuery
286286
}
287287
catch (Exception e)
288288
{
289-
Log.Instance.Fatal("Request params: " + Utility.JsonSerialize_IgnoreLoopingReference(RequestParams));
290-
Log.Instance.Fatal(e);
289+
Log.Instance.Error("Request params: " + Utility.JsonSerialize_IgnoreLoopingReference(RequestParams));
290+
Log.Instance.Error(e);
291291
await ParseError(context, HttpStatusCode.BadRequest, sourceToken, "Bad Request");
292292
}
293293
}

0 commit comments

Comments
 (0)