feat(autocomplete): support autocomplete

.github/workflows/package_push_nuget.org.yml

@@ -10,33 +10,51 @@ jobs:
     steps:
     - name: git pull
       uses: actions/checkout@v2
-     
+
     - name: run a one-line script
       run: env
-     
-    - name: setting dotnet version 
+
+    - name: setting dotnet version
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: '6.0.x'
         include-prerelease: true
     - name: dependencies
       run: git clone -b main https://github.com/masastack/MASA.BuildingBlocks.git ./src/BuildingBlocks/MASA.BuildingBlocks
-
+
+    - name: Configure sysctl limits
+      run: |
+        sudo swapoff -a
+        sudo sysctl -w vm.swappiness=1
+        sudo sysctl -w fs.file-max=262144
+        sudo sysctl -w vm.max_map_count=262144
+
+    - name: Start Elasticsearch
+      uses: everpcpc/elasticsearch-action@v2
+      with:
+        version: 7.6.1
+        plugins: |
+          https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.6.1/elasticsearch-analysis-ik-7.6.1.zip
+          https://github.com/medcl/elasticsearch-analysis-pinyin/releases/download/v7.6.1/elasticsearch-analysis-pinyin-7.6.1.zip
+          analysis-icu
+          analysis-smartcn
+          analysis-kuromoji
+
     - name: restore
       run: dotnet restore
-     
+
     - name: build
       run: dotnet build --no-restore /p:ContinuousIntegrationBuild=true
-     
+
     - name: test
       run: dotnet test --no-build --verbosity normal /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[*.Tests]*"
-     
+
     - name: codecov
       uses: codecov/codecov-action@v1
-     
+
     - name: pack
       run: dotnet pack --include-symbols  -p:PackageVersion=$GITHUB_REF_NAME
-     
+
     - name: package push
       run: dotnet nuget push "**/*.symbols.nupkg" -k  ${{secrets.NUGET_TOKEN}}  -s https://api.nuget.org/v3/index.json
-        
+

Masa.Contrib.sln

@@ -132,6 +132,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.BuildingBlocks.Service
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.Dispatcher.Events.HandlerOrder.Tests", "test\Masa.Contrib.Dispatcher.Events.HandlerOrder.Tests\Masa.Contrib.Dispatcher.Events.HandlerOrder.Tests.csproj", "{4A052E17-4D9E-41EF-89A5-73B917053F8E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.SearchEngine.AutoComplete", "src\SearchEngine\Masa.Contrib.SearchEngine.AutoComplete\Masa.Contrib.SearchEngine.AutoComplete.csproj", "{3F8532EF-3DC9-45F8-9562-994ABE066585}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.SearchEngine.AutoComplete.Tests", "test\Masa.Contrib.SearchEngine.AutoComplete.Tests\Masa.Contrib.SearchEngine.AutoComplete.Tests.csproj", "{31262D61-26A4-4302-968D-52B8DA4558CD}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -484,6 +488,22 @@ Global
 		{4A052E17-4D9E-41EF-89A5-73B917053F8E}.Release|Any CPU.Build.0 = Release|Any CPU
 		{4A052E17-4D9E-41EF-89A5-73B917053F8E}.Release|x64.ActiveCfg = Release|Any CPU
 		{4A052E17-4D9E-41EF-89A5-73B917053F8E}.Release|x64.Build.0 = Release|Any CPU
+		{3F8532EF-3DC9-45F8-9562-994ABE066585}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3F8532EF-3DC9-45F8-9562-994ABE066585}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3F8532EF-3DC9-45F8-9562-994ABE066585}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3F8532EF-3DC9-45F8-9562-994ABE066585}.Debug|x64.Build.0 = Debug|Any CPU
+		{3F8532EF-3DC9-45F8-9562-994ABE066585}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3F8532EF-3DC9-45F8-9562-994ABE066585}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3F8532EF-3DC9-45F8-9562-994ABE066585}.Release|x64.ActiveCfg = Release|Any CPU
+		{3F8532EF-3DC9-45F8-9562-994ABE066585}.Release|x64.Build.0 = Release|Any CPU
+		{31262D61-26A4-4302-968D-52B8DA4558CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{31262D61-26A4-4302-968D-52B8DA4558CD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{31262D61-26A4-4302-968D-52B8DA4558CD}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{31262D61-26A4-4302-968D-52B8DA4558CD}.Debug|x64.Build.0 = Debug|Any CPU
+		{31262D61-26A4-4302-968D-52B8DA4558CD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{31262D61-26A4-4302-968D-52B8DA4558CD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{31262D61-26A4-4302-968D-52B8DA4558CD}.Release|x64.ActiveCfg = Release|Any CPU
+		{31262D61-26A4-4302-968D-52B8DA4558CD}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -549,6 +569,8 @@ Global
 		{2F4986D6-3F56-4C05-8A1D-399594F96093} = {DC578D74-98F0-4F19-A230-CFA8DAEE0AF1}
 		{E72E105D-B15F-4D69-9A13-CAA49D4889D6} = {DC578D74-98F0-4F19-A230-CFA8DAEE0AF1}
 		{4A052E17-4D9E-41EF-89A5-73B917053F8E} = {2BE750A5-8AC7-457C-9BB2-6E3D5E2D23B8}
+		{3F8532EF-3DC9-45F8-9562-994ABE066585} = {8C39C640-0E8A-43A7-890C-9742B6B70AA4}
+		{31262D61-26A4-4302-968D-52B8DA4558CD} = {38E6C400-90C0-493E-9266-C1602E229F1B}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {40383055-CC50-4600-AD9A-53C14F620D03}

src/BasicAbility/Masa.Contrib.BasicAbility.Dcc/Masa.Contrib.BasicAbility.Dcc.csproj

@@ -7,9 +7,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Masa.Utils.Caching.DistributedMemory" Version="0.4.0-preview.1" />
-    <PackageReference Include="Masa.Utils.Caching.Redis" Version="0.4.0-preview.1" />
-    <PackageReference Include="Masa.Utils.Caller.HttpClient" Version="0.4.0-preview.1" />
+    <PackageReference Include="Masa.Utils.Caching.DistributedMemory" Version="0.4.0-preview.2" />
+    <PackageReference Include="Masa.Utils.Caching.Redis" Version="0.4.0-preview.2" />
+    <PackageReference Include="Masa.Utils.Caller.HttpClient" Version="0.4.0-preview.2" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />

src/Data/Masa.Contrib.Data.Contracts.EF/Masa.Contrib.Data.Contracts.EF.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Masa.Utils.Data.EntityFrameworkCore" Version="0.4.0-preview.1" />
+    <PackageReference Include="Masa.Utils.Data.EntityFrameworkCore" Version="0.4.0-preview.2" />
     <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="6.0.0" />
   </ItemGroup>

src/Data/Masa.Contrib.Data.UoW.EF/Masa.Contrib.Data.UoW.EF.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Masa.Utils.Data.EntityFrameworkCore" Version="0.4.0-preview.1" />
+    <PackageReference Include="Masa.Utils.Data.EntityFrameworkCore" Version="0.4.0-preview.2" />
     <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="6.0.0" />
   </ItemGroup>

src/Dispatcher/Masa.Contrib.Dispatcher.Events/Internal/Dispatch/DispatcherBase.cs

@@ -96,9 +96,8 @@ await executionStrategy.ExecuteAsync(strategyOptions, @event, async @event =>
             }, (@event, ex, failureLevels) =>
             {
                 if (failureLevels != FailureLevels.Ignore)
-                {
-                    throw ex;
-                }
+                    throw new Exception(ex.Message, ex);
+
                 logger?.LogWarning("----- Publishing event {@Event} rollback error ignored: message id: {messageId} -----", @event, @event.Id);
                 return Task.CompletedTask;
             });

src/Dispatcher/Masa.Contrib.Dispatcher.Events/Masa.Contrib.Dispatcher.Events.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Masa.Utils.Models.Config" Version="0.4.0-preview.1" />
+    <PackageReference Include="Masa.Utils.Models.Config" Version="0.4.0-preview.2" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />

src/Dispatcher/Masa.Contrib.Dispatcher.IntegrationEvents.Dapr/Masa.Contrib.Dispatcher.IntegrationEvents.Dapr.csproj

@@ -8,8 +8,8 @@
 
   <ItemGroup>
     <PackageReference Include="Dapr.AspNetCore" Version="1.5.0" />
-    <PackageReference Include="Masa.Utils.Models.Config" Version="0.4.0-preview.1" />
-    <PackageReference Include="Masa.Utils.Exceptions" Version="0.4.0-preview.1" />
+    <PackageReference Include="Masa.Utils.Models.Config" Version="0.4.0-preview.2" />
+    <PackageReference Include="Masa.Utils.Exceptions" Version="0.4.0-preview.2" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
   </ItemGroup>

src/Dispatcher/Masa.Contrib.Dispatcher.IntegrationEvents.EventLogs.EF/Masa.Contrib.Dispatcher.IntegrationEvents.EventLogs.EF.csproj

@@ -7,8 +7,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Masa.Utils.Data.EntityFrameworkCore" Version="0.4.0-preview.1" />
-    <PackageReference Include="Masa.Utils.Exceptions" Version="0.4.0-preview.1" />
+    <PackageReference Include="Masa.Utils.Data.EntityFrameworkCore" Version="0.4.0-preview.2" />
+    <PackageReference Include="Masa.Utils.Exceptions" Version="0.4.0-preview.2" />
     <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="6.0.0" />
   </ItemGroup>

src/SearchEngine/Masa.Contrib.SearchEngine.AutoComplete/AutoCompleteClient.cs

@@ -0,0 +1,135 @@
+namespace Masa.Contrib.SearchEngine.AutoComplete;
+
+public class AutoCompleteClient : IAutoCompleteClient
+{
+    private readonly IElasticClient _elasticClient;
+    private readonly IMasaElasticClient _client;
+    private readonly string _indexName;
+    private readonly Operator _defaultOperator;
+    private readonly SearchType _defaultSearchType;
+
+    public AutoCompleteClient(IElasticClient elasticClient, IMasaElasticClient client, string indexName, Operator defaultOperator, SearchType defaultSearchType)
+    {
+        _elasticClient = elasticClient;
+        _client = client;
+        _indexName = indexName;
+        _defaultOperator = defaultOperator;
+        _defaultSearchType = defaultSearchType;
+    }
+
+    public Task<GetResponse<AutoCompleteDocument<TValue>, TValue>> GetAsync<TValue>(string value,
+        AutoCompleteOptions? options = null,
+        CancellationToken cancellationToken = default)
+        => GetAsync<AutoCompleteDocument<TValue>, TValue>(value, options, cancellationToken);
+
+    public async Task<GetResponse<TResponse, TValue>> GetAsync<TResponse, TValue>(string value,
+        AutoCompleteOptions? options = null,
+        CancellationToken cancellationToken = default)
+        where TResponse : AutoCompleteDocument<TValue>
+    {
+        var newOptions = options ?? new (_defaultSearchType);
+        if (newOptions.SearchType == SearchType.Fuzzy)
+        {
+            var ret = await _client.GetPaginatedListAsync(
+                new PaginatedOptions<TResponse>(
+                    _indexName,
+                    value,
+                    newOptions.Field,
+                    newOptions.Page,
+                    newOptions.PageSize,
+                    _defaultOperator)
+                , cancellationToken);
+            return new GetResponse<TResponse, TValue>(ret.IsValid, ret.Message)
+            {
+                Total = ret.Total,
+                TotalPages = ret.TotalPages,
+                Data = ret.Data
+            };
+        }
+        else
+        {
+            var ret = await _elasticClient.SearchAsync<TResponse>(s => s
+                    .Index(_indexName)
+                    .From((newOptions.Page - 1) * newOptions.PageSize)
+                    .Size(newOptions.PageSize)
+                    .Query(q => q
+                        .Bool(b => b
+                            .Must(queryContainerDescriptor => queryContainerDescriptor.Term(newOptions.Field, value))))
+                , cancellationToken
+            );
+            return new GetResponse<TResponse, TValue>(ret.IsValid, ret.ServerError?.ToString() ?? "")
+            {
+                Data = ret.Hits.Select(hit => hit.Source).ToList(),
+                Total = ret.Total,
+                TotalPages = (int)Math.Ceiling(ret.Total / (decimal)newOptions.PageSize)
+            };
+        }
+    }
+
+    public Task<SetResponse> SetAsync<TValue>(
+        AutoCompleteDocument<TValue>[] results,
+        SetOptions? options = null,
+        CancellationToken cancellationToken = default)
+        => SetAsync<AutoCompleteDocument<TValue>, TValue>(results, options, cancellationToken);
+
+    public Task<SetResponse> SetAsync<TDocument, TValue>(
+        TDocument[] documents,
+        SetOptions? options = null,
+        CancellationToken cancellationToken = default) where TDocument : AutoCompleteDocument<TValue>
+    {
+        SetOptions newOptions = options ?? new();
+        if (newOptions.IsOverride)
+            return SetAsync<TDocument, TValue>(documents, cancellationToken);
+
+        return SetByNotOverrideAsync<TDocument, TValue>(documents, cancellationToken);
+    }
+
+    /// <summary>
+    /// Set documents in batches
+    /// add them if they don’t exist, update them if they exist
+    /// </summary>
+    /// <param name="documents"></param>
+    /// <param name="cancellationToken"></param>
+    /// <typeparam name="TDocument"></typeparam>
+    /// <typeparam name="TValue"></typeparam>
+    /// <returns></returns>
+    private async Task<SetResponse> SetAsync<TDocument, TValue>(
+        TDocument[] documents,
+        CancellationToken cancellationToken = default)
+        where TDocument : AutoCompleteDocument<TValue>
+    {
+        var request = new SetDocumentRequest<TDocument>(_indexName);
+        foreach (var document in documents)
+            request.AddDocument(document, document.Id);
+
+        var ret = await _client.SetDocumentAsync(request, cancellationToken);
+        return new SetResponse(ret.IsValid, ret.Message)
+        {
+            Items = ret.Items.Select(item => new SetResponseItems(item.Id, item.IsValid, item.Message)).ToList()
+        };
+    }
+
+    /// <summary>
+    /// Set documents in batches
+    /// Update if it does not exist, skip if it exists
+    /// </summary>
+    /// <param name="documents"></param>
+    /// <param name="cancellationToken"></param>
+    /// <typeparam name="TDocument"></typeparam>
+    /// <typeparam name="TValue"></typeparam>
+    /// <returns></returns>
+    private async Task<SetResponse> SetByNotOverrideAsync<TDocument, TValue>(
+        TDocument[] documents,
+        CancellationToken cancellationToken = default) where TDocument : AutoCompleteDocument<TValue>
+    {
+        var request = new CreateMultiDocumentRequest<TDocument>(_indexName);
+        foreach (var document in documents)
+            request.AddDocument(document, document.Id);
+
+        var ret = await _client.CreateMultiDocumentAsync(request, cancellationToken);
+        return new SetResponse(ret.IsValid, ret.Message)
+        {
+            Items = ret.Items.Select(item => new SetResponseItems(item.Id, item.IsValid, item.Message)).ToList()
+        };
+    }
+}

src/SearchEngine/Masa.Contrib.SearchEngine.AutoComplete/AutoCompleteFactory.cs

@@ -0,0 +1,27 @@
+namespace Masa.Contrib.SearchEngine.AutoComplete;
+
+public class AutoCompleteFactory : IAutoCompleteFactory
+{
+    private readonly List<AutoCompleteRelations> _relations;
+
+    public AutoCompleteFactory(AutoCompleteRelationsOptions options) => _relations = options.Relations;
+
+    public IAutoCompleteClient CreateClient()
+    {
+        var item = _relations.SingleOrDefault(r => r.IsDefault) ?? _relations.FirstOrDefault();
+        ArgumentNullException.ThrowIfNull(item, "You should use AddAutoComplete before the project starts");
+        return new AutoCompleteClient(item.ElasticClient, item.MasaElasticClient, item.RealIndexName, item.DefaultOperator, item.DefaultSearchType);
+    }
+
+    /// <summary>
+    /// Create a client corresponding to the index
+    /// </summary>
+    /// <param name="name">indexName or alias</param>
+    /// <returns></returns>
+    public IAutoCompleteClient CreateClient(string name)
+    {
+        var item = _relations.FirstOrDefault(relation => relation.IndexName == name || relation.Alias == name);
+        ArgumentNullException.ThrowIfNull(item, nameof(name));
+        return new AutoCompleteClient(item.ElasticClient, item.MasaElasticClient, item.RealIndexName, item.DefaultOperator, item.DefaultSearchType);
+    }
+}

src/SearchEngine/Masa.Contrib.SearchEngine.AutoComplete/Masa.Contrib.SearchEngine.AutoComplete.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Masa.Utils.Data.Elasticsearch" Version="0.4.0-preview.2" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\BuildingBlocks\MASA.BuildingBlocks\src\SearchEngine\Masa.BuildingBlocks.SearchEngine.AutoComplete\Masa.BuildingBlocks.SearchEngine.AutoComplete.csproj" />
+  </ItemGroup>
+
+</Project>