Skip to content

Commit 4908727

Browse files
zhenlei520zhenlei520
authored
feat(redis): Support fuzzy query (#173)
* chore: Readjust the directory structure * style: BasicAbility rename to StackSdks * chore: using global version * refactor: Removed using Utils from nuget package * chore: init * ci: remove buildingBlocks dependencies * ci: Adjust gitignore * chore: Add Scenes * chore: remove src by utils * refactor: Refactor repo structure * refactor: Refactor repo structure * refactor: Refactor repo structure * refactor: Refactor repo structure * rename: Oidc rename to OpenIdConnect * rename: Identity.IdentityModel rename to Authentication.Identity * rename: EF rename to EntityFrameworkCore * rename: Benchmark rename to Tests.Benchmark * chore: global using sort * refactor: remove invalid references * chore: Ignore deprecation warnings * chore: use global using * refactor: Remove the restriction that the Query response value cannot be empty * refactor: Refactor repo structure * chore: Modify unit test name * rename: Benchmark rename to Perf * chore: Adjust the directory structure * refactor: Refactor repo structure * feature(Cache): Support fuzzy filtering * feature(Cache): Supports asynchronous collection of key-value pairs * refactor(Caching.Redis): Shorthand code * chore: Delete submodules * style: format code
1 parent 4d81f82 commit 4908727

File tree

5 files changed

+241
-101
lines changed

5 files changed

+241
-101
lines changed

‎.gitmodules

-3
This file was deleted.

‎src/Utils/Caching/Distributed/Masa.Utils.Caching.Redis/RedisCacheClient.cs

+147-98
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ public class RedisCacheClient : IDistributedCacheClient
3333
end
3434
return count";
3535

36+
private const string GET_KEYS_SCRIPT = @"return redis.call('keys', @pattern)";
37+
38+
private const string GET_KEY_AND_VALUE_SCRIPT = @"local ks = redis.call('KEYS', @keypattern)
39+
local result = {}
40+
for index,val in pairs(ks) do result[(2 * index - 1)] = val; result[(2 * index)] = redis.call('hgetall', val) end;
41+
return result";
42+
3643
private const string ABSOLUTE_EXPIRATION_KEY = "absexp";
3744
private const string SLIDING_EXPIRATION_KEY = "sldexp";
3845
private const string DATA_KEY = "data";
@@ -296,6 +303,78 @@ public async Task<bool> ExistsAsync<T>(string key)
296303
return await _db.KeyExistsAsync(key);
297304
}
298305

306+
/// <summary>
307+
/// Support fuzzy filtering to obtain key set
308+
/// </summary>
309+
/// <param name="keyPattern"></param>
310+
/// <returns></returns>
311+
public List<string> GetKeys(string keyPattern)
312+
{
313+
var prepared = LuaScript.Prepare(GET_KEYS_SCRIPT);
314+
var cacheResult = _db.ScriptEvaluate(prepared, new { pattern = keyPattern });
315+
if (cacheResult.IsNull)
316+
return new List<string>();
317+
318+
return ((string[])cacheResult).ToList();
319+
}
320+
321+
/// <summary>
322+
/// Support fuzzy filtering to obtain key set
323+
/// </summary>
324+
/// <param name="keyPattern"></param>
325+
/// <returns></returns>
326+
public async Task<List<string>> GetKeysAsync(string keyPattern)
327+
{
328+
var prepared = LuaScript.Prepare(GET_KEYS_SCRIPT);
329+
var cacheResult = await _db.ScriptEvaluateAsync(prepared, new { pattern = keyPattern });
330+
if (cacheResult.IsNull) return new List<string>();
331+
332+
return ((string[])cacheResult).ToList();
333+
}
334+
335+
public Dictionary<string, T?> GetListByKeyPattern<T>(string keyPattern)
336+
{
337+
var arrayRedisResult = _db.ScriptEvaluate(
338+
LuaScript.Prepare(GET_KEY_AND_VALUE_SCRIPT),
339+
new { keypattern = keyPattern }).ToDictionary();
340+
341+
return GetListByKeyPatternCore<T?>(
342+
arrayRedisResult,
343+
(key, absExpr, sldExpr) =>
344+
{
345+
Refresh(key, absExpr, sldExpr);
346+
return Task.CompletedTask;
347+
});
348+
}
349+
350+
public async Task<Dictionary<string, T?>> GetListByKeyPatternAsync<T>(string keyPattern)
351+
{
352+
var arrayRedisResult = (await _db.ScriptEvaluateAsync(
353+
LuaScript.Prepare(GET_KEY_AND_VALUE_SCRIPT),
354+
new { keypattern = keyPattern })).ToDictionary();
355+
356+
return GetListByKeyPatternCore<T?>(
357+
arrayRedisResult,
358+
async (key, absExpr, sldExpr) => await RefreshAsync(key, absExpr, sldExpr));
359+
}
360+
361+
private Dictionary<string, T?> GetListByKeyPatternCore<T>(
362+
Dictionary<string, RedisResult> arrayRedisResult,
363+
Func<string, DateTimeOffset?, TimeSpan?, Task> func)
364+
{
365+
Dictionary<string, T?> dictionary = new();
366+
367+
foreach (var redisResult in arrayRedisResult)
368+
{
369+
var byteArray = (RedisValue[])redisResult.Value;
370+
MapMetadata(byteArray, out DateTimeOffset? absExpr, out TimeSpan? sldExpr, out RedisValue data);
371+
func.Invoke(redisResult.Key, absExpr, sldExpr);
372+
dictionary.Add(redisResult.Key, ConvertToValue<T>(data));
373+
}
374+
375+
return dictionary;
376+
}
377+
299378
/// <inheritdoc />
300379
public void Refresh(string key)
301380
{
@@ -331,31 +410,25 @@ await _subscriber.SubscribeAsync(channel, (_, message) =>
331410
/// <inheritdoc />
332411
public void Publish<T>(string channel, Action<SubscribeOptions<T>> setup)
333412
{
334-
if (channel == null)
335-
throw new ArgumentNullException(nameof(channel));
336-
337-
if (setup == null)
338-
throw new ArgumentNullException(nameof(setup));
339-
340-
var options = new SubscribeOptions<T>();
341-
setup.Invoke(options);
342-
343-
if (string.IsNullOrWhiteSpace(options.Key))
344-
throw new ArgumentNullException(nameof(options.Key));
345-
346-
var message = JsonSerializer.Serialize(options);
347-
348-
_subscriber.Publish(channel, message);
413+
PublishCore(channel, setup, (c, message) =>
414+
{
415+
_subscriber.Publish(c, message);
416+
return Task.CompletedTask;
417+
});
349418
}
350419

351420
/// <inheritdoc />
352421
public async Task PublishAsync<T>(string channel, Action<SubscribeOptions<T>> setup)
353422
{
354-
if (channel == null)
355-
throw new ArgumentNullException(nameof(channel));
423+
PublishCore(channel, setup, async (c, message) => await _subscriber.PublishAsync(c, message));
424+
await Task.CompletedTask;
425+
}
426+
427+
private void PublishCore<T>(string channel, Action<SubscribeOptions<T>> setup, Func<string, string, Task> func)
428+
{
429+
ArgumentNullException.ThrowIfNull(channel, nameof(channel));
356430

357-
if (setup == null)
358-
throw new ArgumentNullException(nameof(setup));
431+
ArgumentNullException.ThrowIfNull(setup, nameof(setup));
359432

360433
var options = new SubscribeOptions<T>();
361434
setup.Invoke(options);
@@ -364,8 +437,7 @@ public async Task PublishAsync<T>(string channel, Action<SubscribeOptions<T>> se
364437
throw new ArgumentNullException(nameof(options.Key));
365438

366439
var message = JsonSerializer.Serialize(options);
367-
368-
await _subscriber.PublishAsync(channel, message);
440+
func.Invoke(channel, message);
369441
}
370442

371443
public async Task<long> HashIncrementAsync(string key, long value = 1L)
@@ -393,92 +465,55 @@ private RedisValue GetAndRefresh(string key, bool getData)
393465
if (key == null)
394466
throw new ArgumentNullException(nameof(key));
395467

396-
RedisValue[] results;
397-
if (getData)
398-
{
399-
results = _db.HashMemberGet(key, ABSOLUTE_EXPIRATION_KEY, SLIDING_EXPIRATION_KEY, DATA_KEY);
400-
}
401-
else
402-
{
403-
results = _db.HashMemberGet(key, ABSOLUTE_EXPIRATION_KEY, SLIDING_EXPIRATION_KEY);
404-
}
405-
406-
if (results.Length >= 2)
407-
{
408-
MapMetadata(results, out DateTimeOffset? absExpr, out TimeSpan? sldExpr);
409-
Refresh(key, absExpr, sldExpr);
410-
}
468+
var results = getData ? _db.HashMemberGet(key, ABSOLUTE_EXPIRATION_KEY, SLIDING_EXPIRATION_KEY, DATA_KEY) :
469+
_db.HashMemberGet(key, ABSOLUTE_EXPIRATION_KEY, SLIDING_EXPIRATION_KEY);
411470

412-
if (results.Length >= 3 && results[2].HasValue)
413-
{
414-
return results[2];
415-
}
471+
MapMetadata(results, out DateTimeOffset? absExpr, out TimeSpan? sldExpr, out RedisValue data);
472+
Refresh(key, absExpr, sldExpr);
416473

417-
return RedisValue.Null;
474+
return data;
418475
}
419476

420477
private async Task<RedisValue> GetAndRefreshAsync(string key, bool getData)
421478
{
422479
if (key == null)
423480
throw new ArgumentNullException(nameof(key));
424481

425-
RedisValue[] results;
426-
if (getData)
427-
{
428-
results = await _db.HashMemberGetAsync(key, ABSOLUTE_EXPIRATION_KEY, SLIDING_EXPIRATION_KEY, DATA_KEY).ConfigureAwait(false);
429-
}
430-
else
431-
{
432-
results = await _db.HashMemberGetAsync(key, ABSOLUTE_EXPIRATION_KEY, SLIDING_EXPIRATION_KEY).ConfigureAwait(false);
433-
}
434-
435-
// TODO: Error handling
436-
if (results.Length >= 2)
437-
{
438-
MapMetadata(results, out DateTimeOffset? absExpr, out TimeSpan? sldExpr);
439-
await RefreshAsync(key, absExpr, sldExpr).ConfigureAwait(false);
440-
}
482+
var results = getData ?
483+
await _db.HashMemberGetAsync(key, ABSOLUTE_EXPIRATION_KEY, SLIDING_EXPIRATION_KEY, DATA_KEY).ConfigureAwait(false) :
484+
await _db.HashMemberGetAsync(key, ABSOLUTE_EXPIRATION_KEY, SLIDING_EXPIRATION_KEY).ConfigureAwait(false);
441485

442-
if (results.Length >= 3 && results[2].HasValue)
443-
{
444-
return results[2];
445-
}
446-
447-
return RedisValue.Null;
486+
MapMetadata(results, out DateTimeOffset? absExpr, out TimeSpan? sldExpr, out var data);
487+
await RefreshAsync(key, absExpr, sldExpr).ConfigureAwait(false);
488+
return data;
448489
}
449490

450491
private void Refresh(string key, DateTimeOffset? absExpr, TimeSpan? sldExpr)
451492
{
452-
if (key == null)
453-
{
454-
throw new ArgumentNullException(nameof(key));
455-
}
456-
457-
// Note Refresh has no effect if there is just an absolute expiration (or neither).
458-
if (sldExpr.HasValue)
493+
RefreshCore(key, absExpr, sldExpr, (k, expr) =>
459494
{
460-
TimeSpan? expr;
461-
if (absExpr.HasValue)
462-
{
463-
var relExpr = absExpr.Value - DateTimeOffset.Now;
464-
expr = relExpr <= sldExpr.Value ? relExpr : sldExpr;
465-
}
466-
else
467-
{
468-
expr = sldExpr;
469-
}
470-
471-
_db.KeyExpire(key, expr);
472-
// TODO: Error handling
473-
}
495+
_db.KeyExpire(k, expr);
496+
return Task.CompletedTask;
497+
});
474498
}
475499

476500
private async Task RefreshAsync(string key, DateTimeOffset? absExpr, TimeSpan? sldExpr, CancellationToken token = default)
477501
{
478-
if (key == null)
502+
RefreshCore(key, absExpr, sldExpr, async (k, expr) =>
479503
{
480-
throw new ArgumentNullException(nameof(key));
481-
}
504+
await _db.KeyExpireAsync(k, expr).ConfigureAwait(false);
505+
}, token);
506+
await Task.CompletedTask;
507+
}
508+
509+
private void RefreshCore(
510+
string key,
511+
DateTimeOffset? absExpr,
512+
TimeSpan? sldExpr,
513+
Func<string, TimeSpan?, Task> func,
514+
CancellationToken token = default)
515+
{
516+
ArgumentNullException.ThrowIfNull(key, nameof(key));
482517

483518
token.ThrowIfCancellationRequested();
484519

@@ -496,25 +531,39 @@ private async Task RefreshAsync(string key, DateTimeOffset? absExpr, TimeSpan? s
496531
expr = sldExpr;
497532
}
498533

499-
await _db.KeyExpireAsync(key, expr).ConfigureAwait(false);
500-
// TODO: Error handling
534+
func.Invoke(key, expr);
501535
}
502536
}
503537

504-
private static void MapMetadata(RedisValue[] results, out DateTimeOffset? absoluteExpiration, out TimeSpan? slidingExpiration)
538+
private static void MapMetadata(RedisValue[] results, out DateTimeOffset? absoluteExpiration, out TimeSpan? slidingExpiration,
539+
out RedisValue data)
505540
{
506541
absoluteExpiration = null;
507542
slidingExpiration = null;
508-
var absoluteExpirationTicks = (long?)results[0];
509-
if (absoluteExpirationTicks.HasValue && absoluteExpirationTicks.Value != NOT_PRESENT)
510-
{
511-
absoluteExpiration = new DateTimeOffset(absoluteExpirationTicks.Value, TimeSpan.Zero);
512-
}
543+
data = RedisValue.Null;
513544

514-
var slidingExpirationTicks = (long?)results[1];
515-
if (slidingExpirationTicks.HasValue && slidingExpirationTicks.Value != NOT_PRESENT)
545+
for (int index = 0; index < results.Length; index += 2)
516546
{
517-
slidingExpiration = new TimeSpan(slidingExpirationTicks.Value);
547+
if (results[index] == ABSOLUTE_EXPIRATION_KEY)
548+
{
549+
var absoluteExpirationTicks = (long?)results[index + 1];
550+
if (absoluteExpirationTicks.HasValue && absoluteExpirationTicks.Value != NOT_PRESENT)
551+
{
552+
absoluteExpiration = new DateTimeOffset(absoluteExpirationTicks.Value, TimeSpan.Zero);
553+
}
554+
}
555+
else if (results[index] == SLIDING_EXPIRATION_KEY)
556+
{
557+
var slidingExpirationTicks = (long?)results[index + 1];
558+
if (slidingExpirationTicks.HasValue && slidingExpirationTicks.Value != NOT_PRESENT)
559+
{
560+
slidingExpiration = new TimeSpan(slidingExpirationTicks.Value);
561+
}
562+
}
563+
else if (results[index] == DATA_KEY)
564+
{
565+
data = results[index + 1];
566+
}
518567
}
519568
}
520569

‎src/Utils/Caching/Distributed/Tests/Masa.Utils.Caching.Redis.Tests/DistributedCacheClientTest.cs

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// Copyright (c) MASA Stack All rights reserved.
2+
// Licensed under the MIT License. See LICENSE.txt in the project root for license information.
3+
4+
namespace Masa.Utils.Caching.Redis.Tests;
5+
6+
[TestClass]
7+
public class DistributedCacheClientTest
8+
{
9+
private IServiceProvider _serviceProvider;
10+
private IDistributedCacheClient _cacheClient;
11+
12+
[TestInitialize]
13+
public void Initialize()
14+
{
15+
var services = new ServiceCollection();
16+
services.AddMasaRedisCache(option =>
17+
{
18+
option.Servers = new List<RedisServerOptions>()
19+
{
20+
new("localhost", 6379),
21+
};
22+
});
23+
_serviceProvider = services.BuildServiceProvider();
24+
_cacheClient = _serviceProvider.GetRequiredService<IDistributedCacheClient>();
25+
_cacheClient.Remove<string>("test1", "test2", "redis1", "redis2");
26+
}
27+
28+
[TestMethod]
29+
public async Task TestGetKeysAsyncReturnCountIs2()
30+
{
31+
Assert.IsNotNull(_cacheClient);
32+
_cacheClient.Set("test1", "test1");
33+
_cacheClient.Set("test2", "test2");
34+
_cacheClient.Set("redis1", "redis1");
35+
_cacheClient.Set("redis2", "redis2");
36+
var keys = await _cacheClient.GetKeysAsync("test*");
37+
Assert.AreEqual(2, keys.Count);
38+
}
39+
40+
[TestMethod]
41+
public void TestGetListByKeyPatternReturnCountIs1()
42+
{
43+
_cacheClient.Set("test1", "test1:Result");
44+
_cacheClient.Set("redis1", "redis1");
45+
_cacheClient.Set("redis2", "redis2");
46+
var dictionary = _cacheClient.GetListByKeyPattern<string>("test*");
47+
Assert.AreEqual(1, dictionary.Count);
48+
Assert.IsTrue(dictionary["test1"] == "test1:Result");
49+
}
50+
51+
[TestMethod]
52+
public async Task TestGetListByKeyPatternAsyncReturnCountIs1()
53+
{
54+
_cacheClient.Set("test1", "test1:Result");
55+
_cacheClient.Set("redis1", "redis1");
56+
_cacheClient.Set("redis2", "redis2");
57+
var dictionary = await _cacheClient.GetListByKeyPatternAsync<string>("test*");
58+
Assert.AreEqual(1, dictionary.Count);
59+
Assert.IsTrue(dictionary["test1"] == "test1:Result");
60+
}
61+
}

‎src/Utils/Caching/Distributed/Tests/Masa.Utils.Caching.Redis.Tests/_Imports.cs

+3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
// Copyright (c) MASA Stack All rights reserved.
22
// Licensed under the MIT License. See LICENSE.txt in the project root for license information.
33

4+
global using Masa.Utils.Caching.Core.Interfaces;
45
global using Masa.Utils.Caching.Redis.Helpers;
6+
global using Masa.Utils.Caching.Redis.Models;
7+
global using Microsoft.Extensions.DependencyInjection;
58
global using Microsoft.VisualStudio.TestTools.UnitTesting;
69
global using StackExchange.Redis;
710
global using System;

0 commit comments

Comments
 (0)