From 644fc437b2f9bec0565b6fb3f2f79db6fc27214e Mon Sep 17 00:00:00 2001 From: Matteo Bortolazzo Date: Mon, 14 Sep 2020 14:01:59 +0200 Subject: [PATCH] #102: Ability to create an index --- LATEST_CHANGE.md | 1 + src/.editorconfig | 3 + .../ChangesFeedFilterExtensions.cs | 20 +-- src/CouchDB.Driver/CouchDatabase.cs | 50 +++++- .../Extensions/ExpressionExtensions.cs | 39 +++++ .../Extensions/FlurlRequestExtensions.cs | 1 - src/CouchDB.Driver/ICouchDatabase.cs | 16 +- src/CouchDB.Driver/Indexes/IIndexBuilder.cs | 35 ++++ src/CouchDB.Driver/Indexes/IndexBuilder.cs | 157 ++++++++++++++++++ src/CouchDB.Driver/Indexes/IndexOptions.cs | 8 + .../Options/DocumentCaseType.cs | 2 - .../Translators/MemberExpressionTranslator.cs | 31 +--- tests/CouchDB.Driver.UnitTests/Index_Tests.cs | 55 ++++++ 13 files changed, 367 insertions(+), 51 deletions(-) create mode 100644 src/CouchDB.Driver/Indexes/IIndexBuilder.cs create mode 100644 src/CouchDB.Driver/Indexes/IndexBuilder.cs create mode 100644 src/CouchDB.Driver/Indexes/IndexOptions.cs create mode 100644 tests/CouchDB.Driver.UnitTests/Index_Tests.cs diff --git a/LATEST_CHANGE.md b/LATEST_CHANGE.md index d5649fe..0242b64 100644 --- a/LATEST_CHANGE.md +++ b/LATEST_CHANGE.md @@ -1,4 +1,5 @@ ## Features +* **Indexes"**: Ability to create indexes. * **Null values"**: New `SetNullValueHandling` method for `CouchOptionsBuilder` to set how to handle null values. ## Bug Fixes diff --git a/src/.editorconfig b/src/.editorconfig index 671c781..d2a67dc 100644 --- a/src/.editorconfig +++ b/src/.editorconfig @@ -147,3 +147,6 @@ csharp_preserve_single_line_statements = true # CA1303: Do not pass literals as localized parameters dotnet_diagnostic.CA1303.severity = silent + +# CA1308: Normalize strings to uppercase +dotnet_diagnostic.CA1308.severity = suggestion \ No newline at end of file diff --git a/src/CouchDB.Driver/ChangesFeed/ChangesFeedFilterExtensions.cs b/src/CouchDB.Driver/ChangesFeed/ChangesFeedFilterExtensions.cs index 31671b4..0d006fc 100644 --- a/src/CouchDB.Driver/ChangesFeed/ChangesFeedFilterExtensions.cs +++ b/src/CouchDB.Driver/ChangesFeed/ChangesFeedFilterExtensions.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Linq; using System.Linq.Expressions; using System.Net.Http; using System.Threading; @@ -8,7 +7,6 @@ using CouchDB.Driver.ChangesFeed.Filters; using CouchDB.Driver.ChangesFeed.Responses; using CouchDB.Driver.Extensions; -using CouchDB.Driver.Options; using CouchDB.Driver.Query; using CouchDB.Driver.Types; using Flurl.Http; @@ -17,7 +15,7 @@ namespace CouchDB.Driver.ChangesFeed { internal static class ChangesFeedFilterExtensions { - public static async Task> QueryWithFilterAsync(this IFlurlRequest request, CouchOptions options, ChangesFeedFilter filter, + public static async Task> QueryWithFilterAsync(this IFlurlRequest request, IAsyncQueryProvider queryProvider, ChangesFeedFilter filter, CancellationToken cancellationToken) where TSource : CouchDocument { @@ -32,12 +30,9 @@ public static async Task> QueryWithFilterAsync selectorFilter) { - MethodCallExpression whereExpression = Expression.Call(typeof(Queryable), nameof(Queryable.Where), - new[] { typeof(TSource) }, Expression.Constant(Array.Empty().AsQueryable()), selectorFilter.Value); + MethodCallExpression whereExpression = selectorFilter.Value.WrapInWhereExpression(); + var jsonSelector = queryProvider.ToString(whereExpression); - var optimizer = new QueryOptimizer(); - Expression optimizedQuery = optimizer.Optimize(whereExpression); - var jsonSelector = new QueryTranslator(options).Translate(optimizedQuery); return await request .WithHeader("Content-Type", "application/json") .SetQueryParam("filter", "_selector") @@ -65,7 +60,7 @@ public static async Task> QueryWithFilterAsync QueryContinuousWithFilterAsync(this IFlurlRequest request, CouchOptions options, ChangesFeedFilter filter, CancellationToken cancellationToken) + public static async Task QueryContinuousWithFilterAsync(this IFlurlRequest request, IAsyncQueryProvider queryProvider, ChangesFeedFilter filter, CancellationToken cancellationToken) where TSource: CouchDocument { if (filter is DocumentIdsChangesFeedFilter documentIdsFilter) @@ -78,12 +73,9 @@ public static async Task QueryContinuousWithFilterAsync(this IF if (filter is SelectorChangesFeedFilter selectorFilter) { - MethodCallExpression whereExpression = Expression.Call(typeof(Queryable), nameof(Queryable.Where), - new[] { typeof(TSource) }, Expression.Constant(Array.Empty().AsQueryable()), selectorFilter.Value); + MethodCallExpression whereExpression = selectorFilter.Value.WrapInWhereExpression(); + var jsonSelector = queryProvider.ToString(whereExpression); - var optimizer = new QueryOptimizer(); - Expression optimizedQuery = optimizer.Optimize(whereExpression); - var jsonSelector = new QueryTranslator(options).Translate(optimizedQuery); return await request .WithHeader("Content-Type", "application/json") .SetQueryParam("filter", "_selector") diff --git a/src/CouchDB.Driver/CouchDatabase.cs b/src/CouchDB.Driver/CouchDatabase.cs index a3c7158..1a890e8 100644 --- a/src/CouchDB.Driver/CouchDatabase.cs +++ b/src/CouchDB.Driver/CouchDatabase.cs @@ -8,19 +8,22 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Linq.Expressions; -using System.Net; using System.Net.Http; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; using CouchDB.Driver.ChangesFeed; using CouchDB.Driver.ChangesFeed.Responses; +using CouchDB.Driver.Indexes; using CouchDB.Driver.Local; using CouchDB.Driver.Options; using CouchDB.Driver.Query; +using Flurl.Util; using Newtonsoft.Json; namespace CouchDB.Driver @@ -374,7 +377,7 @@ public async Task> GetChangesAsync(ChangesFeedOptio return filter == null ? await request.GetJsonAsync>(cancellationToken) .ConfigureAwait(false) - : await request.QueryWithFilterAsync(_options, filter, cancellationToken) + : await request.QueryWithFilterAsync(_queryProvider, filter, cancellationToken) .ConfigureAwait(false); } @@ -396,7 +399,7 @@ public async IAsyncEnumerable> GetContinuousC await using Stream stream = filter == null ? await request.GetStreamAsync(cancellationToken, HttpCompletionOption.ResponseHeadersRead) .ConfigureAwait(false) - : await request.QueryContinuousWithFilterAsync(_options, filter, cancellationToken) + : await request.QueryContinuousWithFilterAsync(_queryProvider, filter, cancellationToken) .ConfigureAwait(false); await foreach (var line in stream.ReadLinesAsync(cancellationToken)) @@ -413,6 +416,45 @@ public async IAsyncEnumerable> GetContinuousC #endregion + #region Index + + public async Task CreateIndexAsync(string name, Action> indexBuilderAction, IndexOptions? options = null) + { + Check.NotNull(name, nameof(name)); + Check.NotNull(indexBuilderAction, nameof(indexBuilderAction)); + + var builder = new IndexBuilder(_options, _queryProvider); + indexBuilderAction(builder); + + var indexJson = builder.ToString(); + + var sb = new StringBuilder(); + sb.Append("{") + .Append($"\"index\":{indexJson},") + .Append($"\"name\":\"{name}\",") + .Append("\"type\":\"json\""); + + if (options?.DesignDocument != null) + { + sb.Append($",\"ddoc\":\"{options.DesignDocument}\""); + } + if (options?.Partitioned != null) + { + sb.Append($",\"partitioned\":{options.Partitioned.ToString().ToLowerInvariant()}"); + } + + sb.Append("}"); + + var request = sb.ToString(); + + await NewRequest().AppendPathSegment("_index") + .PostStringAsync(request) + .SendRequestAsync() + .ConfigureAwait(false); + } + + #endregion + #region Utils /// @@ -495,4 +537,4 @@ internal CouchQueryable AsQueryable() #endregion } -} +} \ No newline at end of file diff --git a/src/CouchDB.Driver/Extensions/ExpressionExtensions.cs b/src/CouchDB.Driver/Extensions/ExpressionExtensions.cs index 5e3cd08..0a21209 100644 --- a/src/CouchDB.Driver/Extensions/ExpressionExtensions.cs +++ b/src/CouchDB.Driver/Extensions/ExpressionExtensions.cs @@ -1,11 +1,43 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Linq.Expressions; +using CouchDB.Driver.Options; namespace CouchDB.Driver.Extensions { internal static class ExpressionExtensions { + public static MemberExpression ToMemberExpression(this Expression selector) + { + if (!(selector is LambdaExpression l) || !(l.Body is MemberExpression m)) + { + throw new ArgumentException("The given expression does not select a property.", nameof(selector)); + } + + return m; + } + + public static string GetPropertyName(this MemberExpression m, CouchOptions options) + { + PropertyCaseType caseType = options.PropertiesCase; + + var members = new List { m.Member.GetCouchPropertyName(caseType) }; + + Expression currentExpression = m.Expression; + + while (currentExpression is MemberExpression cm) + { + members.Add(cm.Member.GetCouchPropertyName(caseType)); + currentExpression = cm.Expression; + } + + members.Reverse(); + var propName = string.Join(".", members.ToArray()); + + return propName; + } + public static bool ContainsSelector(this Expression expression) => expression is MethodCallExpression m && m.Arguments.Count == 2 && m.Arguments[1].IsSelectorExpression(); @@ -39,6 +71,13 @@ public static Expression WrapInLambda(this Expression body, IReadOnlyCollection< return Expression.Quote(lambdaExpression); } + public static MethodCallExpression WrapInWhereExpression(this Expression> selector) + { + MethodCallExpression whereExpression = Expression.Call(typeof(Queryable), nameof(Queryable.Where), + new[] { typeof(TSource) }, Expression.Constant(Array.Empty().AsQueryable()), selector); + return whereExpression; + } + private static Expression StripQuotes(this Expression e) { while (e.NodeType == ExpressionType.Quote) diff --git a/src/CouchDB.Driver/Extensions/FlurlRequestExtensions.cs b/src/CouchDB.Driver/Extensions/FlurlRequestExtensions.cs index 616fafc..f83a5a4 100644 --- a/src/CouchDB.Driver/Extensions/FlurlRequestExtensions.cs +++ b/src/CouchDB.Driver/Extensions/FlurlRequestExtensions.cs @@ -45,7 +45,6 @@ public static Task PostStringStreamAsync( return request.SendAsync(HttpMethod.Post, capturedStringContent, cancellationToken, completionOption).ReceiveStream(); } - [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "")] public static IFlurlRequest ApplyQueryParametersOptions(this IFlurlRequest request, object options) { IEnumerable<(string Name, object? Value)> queryParameters = OptionsHelper.ToQueryParameters(options); diff --git a/src/CouchDB.Driver/ICouchDatabase.cs b/src/CouchDB.Driver/ICouchDatabase.cs index 7c08239..82970fc 100644 --- a/src/CouchDB.Driver/ICouchDatabase.cs +++ b/src/CouchDB.Driver/ICouchDatabase.cs @@ -1,9 +1,11 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using CouchDB.Driver.ChangesFeed; using CouchDB.Driver.ChangesFeed.Responses; +using CouchDB.Driver.Indexes; using CouchDB.Driver.Local; using CouchDB.Driver.Security; using CouchDB.Driver.Types; @@ -124,7 +126,17 @@ IAsyncEnumerable> GetContinuousChangesAsync( CancellationToken cancellationToken); /// - /// Asynchronously downloads a specific attachment. + /// Creates an index for the current database with the given configuration. + /// + /// The name of the index. + /// The action to configure the index builder. + /// The index options. + /// A task that represents the asynchronous operation. + Task CreateIndexAsync(string name, Action> indexBuilderAction, + IndexOptions? options = null); + + /// + /// Asynchronously downloads a specific attachment. /// /// The attachment to download. /// Path of local folder where file is to be downloaded. diff --git a/src/CouchDB.Driver/Indexes/IIndexBuilder.cs b/src/CouchDB.Driver/Indexes/IIndexBuilder.cs new file mode 100644 index 0000000..d0467ff --- /dev/null +++ b/src/CouchDB.Driver/Indexes/IIndexBuilder.cs @@ -0,0 +1,35 @@ +using System; +using System.Linq.Expressions; +using CouchDB.Driver.Types; + +namespace CouchDB.Driver.Indexes +{ + public interface IIndexBuilder + where TSource : CouchDocument + { + IMultiFieldIndexBuilder IndexBy(Expression> selector); + } + + public interface IMultiFieldIndexBuilder : IIndexBuilder + where TSource : CouchDocument + { + IMultiFieldIndexBuilder AlsoBy(Expression> selector); + IMultiFieldIndexBuilder Where(Expression> selector); + IOrderedIndexBuilder OrderBy(Expression> selector); + IOrderedDescendingIndexBuilder OrderByDescending(Expression> selector); + IMultiFieldIndexBuilder Take(int take); + IMultiFieldIndexBuilder Skip(int skip); + } + + public interface IOrderedIndexBuilder : IMultiFieldIndexBuilder + where TSource : CouchDocument + { + IOrderedIndexBuilder ThenBy(Expression> selector); + } + + public interface IOrderedDescendingIndexBuilder : IMultiFieldIndexBuilder + where TSource : CouchDocument + { + IOrderedDescendingIndexBuilder ThenByDescending(Expression> selector); + } +} \ No newline at end of file diff --git a/src/CouchDB.Driver/Indexes/IndexBuilder.cs b/src/CouchDB.Driver/Indexes/IndexBuilder.cs new file mode 100644 index 0000000..9b8a60e --- /dev/null +++ b/src/CouchDB.Driver/Indexes/IndexBuilder.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using CouchDB.Driver.Extensions; +using CouchDB.Driver.Options; +using CouchDB.Driver.Query; +using CouchDB.Driver.Types; + +namespace CouchDB.Driver.Indexes +{ + internal class IndexBuilder: IOrderedIndexBuilder, IOrderedDescendingIndexBuilder + where TSource : CouchDocument + { + private readonly CouchOptions _options; + private readonly IAsyncQueryProvider _queryProvider; + + private bool _ascending = true; + private readonly List _fields; + private readonly List _fieldsOrder; + private int? _toTake; + private int? _toSkip; + private string? _selector; + + public IndexBuilder(CouchOptions options, IAsyncQueryProvider queryProvider) + { + _options = options; + _queryProvider = queryProvider; + _fields = new List(); + _fieldsOrder = new List(); + } + + public IMultiFieldIndexBuilder IndexBy(Expression> selector) + { + var m = selector.ToMemberExpression(); + _fields.Clear(); + _fields.Add(m.GetPropertyName(_options)); + return this; + } + + public IMultiFieldIndexBuilder Where(Expression> selector) + { + MethodCallExpression whereExpression = selector.WrapInWhereExpression(); + var jsonSelector = _queryProvider.ToString(whereExpression); + _selector = jsonSelector.Substring(1, jsonSelector.Length - 2); + return this; + } + + public IOrderedIndexBuilder OrderBy(Expression> selector) + { + var m = selector.ToMemberExpression(); + _ascending = true; + _fieldsOrder.Clear(); + _fieldsOrder.Add(m.GetPropertyName(_options)); + return this; + } + + public IOrderedDescendingIndexBuilder OrderByDescending(Expression> selector) + { + var m = selector.ToMemberExpression(); + _ascending = false; + _fieldsOrder.Clear(); + _fieldsOrder.Add(m.GetPropertyName(_options)); + return this; + } + + public IMultiFieldIndexBuilder Take(int take) + { + _toTake = take; + return this; + } + + public IMultiFieldIndexBuilder Skip(int skip) + { + _toSkip = skip; + return this; + } + + public IMultiFieldIndexBuilder AlsoBy(Expression> selector) + { + var m = selector.ToMemberExpression(); + _fields.Add(m.GetPropertyName(_options)); + return this; + } + + IOrderedIndexBuilder IOrderedIndexBuilder.ThenBy(Expression> selector) + { + var m = selector.ToMemberExpression(); + _fieldsOrder.Add(m.GetPropertyName(_options)); + return this; + } + + IOrderedDescendingIndexBuilder IOrderedDescendingIndexBuilder.ThenByDescending(Expression> selector) + { + var m = selector.ToMemberExpression(); + _fieldsOrder.Add(m.GetPropertyName(_options)); + return this; + } + + public override string ToString() + { + var sb = new StringBuilder(); + + sb.Append("{"); + + // Selector + if (_selector != null) + { + sb.Append(_selector); + sb.Append(","); + } + + // Fields + sb.Append("\"fields\":["); + foreach (var field in _fields) + { + sb.Append($"\"{field}\","); + } + + sb.Length--; + sb.Append("],"); + + // Sort + if (_fieldsOrder.Any()) + { + sb.Append("\"sort\":["); + var order = _ascending ? "asc" : "desc"; + + foreach (var field in _fieldsOrder) + { + sb.Append($"{{\"{field}\":\"{order}\"}},"); + } + + sb.Length--; + sb.Append("],"); + } + + // Limit + if (_toTake != null) + { + sb.Append($"\"limit\":{_toTake},"); + } + + // Skip + if (_toSkip != null) + { + sb.Append($"\"skip\":{_toSkip},"); + } + + sb.Length--; + sb.Append("}"); + + return sb.ToString(); + } + } +} \ No newline at end of file diff --git a/src/CouchDB.Driver/Indexes/IndexOptions.cs b/src/CouchDB.Driver/Indexes/IndexOptions.cs new file mode 100644 index 0000000..84a0217 --- /dev/null +++ b/src/CouchDB.Driver/Indexes/IndexOptions.cs @@ -0,0 +1,8 @@ +namespace CouchDB.Driver.Indexes +{ + public class IndexOptions + { + public string? DesignDocument { get; set; } + public bool? Partitioned { get; set; } + } +} \ No newline at end of file diff --git a/src/CouchDB.Driver/Options/DocumentCaseType.cs b/src/CouchDB.Driver/Options/DocumentCaseType.cs index 5e5ef39..eea4bc6 100644 --- a/src/CouchDB.Driver/Options/DocumentCaseType.cs +++ b/src/CouchDB.Driver/Options/DocumentCaseType.cs @@ -24,7 +24,6 @@ private DocumentCaseType(string value) : base(value) { } /// public static readonly DocumentCaseType KebabCase = new DocumentCaseType("KebabCase"); -#pragma warning disable CA1308 // Normalize strings to uppercase internal override string Convert(string str) { if (Equals(this, None)) @@ -41,6 +40,5 @@ internal override string Convert(string str) } throw new NotSupportedException($"Value {Value} not supported."); } -#pragma warning restore CA1308 // Normalize strings to uppercase } } \ No newline at end of file diff --git a/src/CouchDB.Driver/Query/Translators/MemberExpressionTranslator.cs b/src/CouchDB.Driver/Query/Translators/MemberExpressionTranslator.cs index 257761b..4d7e7b7 100644 --- a/src/CouchDB.Driver/Query/Translators/MemberExpressionTranslator.cs +++ b/src/CouchDB.Driver/Query/Translators/MemberExpressionTranslator.cs @@ -1,7 +1,5 @@ -using Newtonsoft.Json; -using System.Collections.Generic; -using System.Linq.Expressions; -using System.Reflection; +using System.Linq.Expressions; +using CouchDB.Driver.Extensions; namespace CouchDB.Driver.Query { @@ -9,30 +7,7 @@ internal partial class QueryTranslator { protected override Expression VisitMember(MemberExpression m) { - string GetPropertyName(MemberInfo memberInfo) - { - var jsonPropertyAttributes = memberInfo.GetCustomAttributes(typeof(JsonPropertyAttribute), true); - JsonPropertyAttribute? jsonProperty = jsonPropertyAttributes.Length > 0 - ? jsonPropertyAttributes[0] as JsonPropertyAttribute - : null; - - return jsonProperty != null - ? jsonProperty.PropertyName - : _options.PropertiesCase.Convert(memberInfo.Name); - } - - var members = new List { GetPropertyName(m.Member) }; - - Expression currentExpression = m.Expression; - - while (currentExpression is MemberExpression cm) - { - members.Add(GetPropertyName(cm.Member)); - currentExpression = cm.Expression; - } - - members.Reverse(); - var propName = string.Join(".", members.ToArray()); + var propName = m.GetPropertyName(_options); _sb.Append($"\"{propName}\""); return m; diff --git a/tests/CouchDB.Driver.UnitTests/Index_Tests.cs b/tests/CouchDB.Driver.UnitTests/Index_Tests.cs new file mode 100644 index 0000000..d4c6fc8 --- /dev/null +++ b/tests/CouchDB.Driver.UnitTests/Index_Tests.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using CouchDB.Driver.Indexes; +using CouchDB.Driver.UnitTests.Models; +using Flurl.Http.Testing; +using Xunit; + +namespace CouchDB.Driver.UnitTests +{ + public class Index_Tests + { + private readonly ICouchClient _client; + private readonly ICouchDatabase _rebels; + + public Index_Tests() + { + _client = new CouchClient("http://localhost"); + _rebels = _client.GetDatabase(); + } + + [Fact] + public async Task CreateIndexAsync() + { + using var httpTest = new HttpTest(); + httpTest.RespondWithJson(new + { + result = "created" + }); + + await _rebels.CreateIndexAsync("skywalkers", b => b + .IndexBy(r => r.Surname) + .AlsoBy(r => r.Name) + .Where(r => r.Surname == "Skywalker" && r.Age > 3) + .OrderByDescending(r => r.Surname) + .ThenByDescending(r => r.Name) + .Take(100) + .Skip(1), + new IndexOptions() + { + DesignDocument = "skywalkers_ddoc", + Partitioned = true + }); + + + var expectedBody = + "{\"index\":{\"selector\":{\"$and\":[{\"surname\":\"Skywalker\"},{\"age\":{\"$gt\":3}}]},\"fields\":[\"surname\",\"name\"],\"sort\":[{\"surname\":\"desc\"},{\"name\":\"desc\"}],\"limit\":100,\"skip\":1},\"name\":\"skywalkers\",\"type\":\"json\",\"ddoc\":\"skywalkers_ddoc\",\"partitioned\":true}"; + httpTest + .ShouldHaveCalled("http://localhost/rebels/_index") + .WithRequestBody(expectedBody) + .WithVerb(HttpMethod.Post); + } + } +}