diff --git a/docs/documents/querying/linq/projections.md b/docs/documents/querying/linq/projections.md index 957b1ef263..fdb8bf160f 100644 --- a/docs/documents/querying/linq/projections.md +++ b/docs/documents/querying/linq/projections.md @@ -50,7 +50,7 @@ public void use_select_with_multiple_fields_to_other_type() }); } ``` -snippet source | anchor +snippet source | anchor When you wish to retrieve certain properties and transform them into an anonymous type: @@ -74,7 +74,7 @@ public void use_select_to_transform_to_an_anonymous_type() .ShouldHaveTheSameElementsAs("Bill", "Hank", "Sam", "Tom"); } ``` -snippet source | anchor +snippet source | anchor Marten also allows you to run projection queries on deep (nested) properties: @@ -96,7 +96,7 @@ public void transform_with_deep_properties() actual.ShouldHaveTheSameElementsAs(expected); } ``` -snippet source | anchor +snippet source | anchor ## Chaining other Linq Methods @@ -121,7 +121,7 @@ public void use_select_to_another_type_with_first() ?.Name.ShouldBe("Bill"); } ``` -snippet source | anchor +snippet source | anchor ## SelectMany() diff --git a/docs/events/projections/async-daemon.md b/docs/events/projections/async-daemon.md index 02a68c6051..287ced9d84 100644 --- a/docs/events/projections/async-daemon.md +++ b/docs/events/projections/async-daemon.md @@ -214,7 +214,30 @@ projections are caught up to the latest events posted at the time of the call. You can see the usage below from one of the Marten tests where we use that method to just wait until the running projection daemon has caught up: -snippet: sample_using_WaitForNonStaleProjectionDataAsync + + +```cs +[Fact] +public async Task run_simultaneously() +{ + StoreOptions(x => x.Projections.Add(new DistanceProjection(), ProjectionLifecycle.Async)); + + NumberOfStreams = 10; + + var agent = await StartDaemon(); + + // This method publishes a random number of events + await PublishSingleThreaded(); + + // Wait for all projections to reach the highest event sequence point + // as of the time this method is called + await theStore.WaitForNonStaleProjectionDataAsync(15.Seconds()); + + await CheckExpectedResults(); +} +``` +snippet source | anchor + The basic idea in your tests is to: diff --git a/docs/events/projections/rebuilding.md b/docs/events/projections/rebuilding.md index ebb212e0ee..4270a88e78 100644 --- a/docs/events/projections/rebuilding.md +++ b/docs/events/projections/rebuilding.md @@ -20,7 +20,7 @@ public class DistanceProjection: EventProjection } } ``` -snippet source | anchor +snippet source | anchor @@ -36,5 +36,5 @@ await PublishSingleThreaded(); // rebuild projection `Distance` await agent.RebuildProjection("Distance", CancellationToken.None); ``` -snippet source | anchor +snippet source | anchor diff --git a/src/LinqTests/Acceptance/Support/LinqTestContext.cs b/src/LinqTests/Acceptance/Support/LinqTestContext.cs index 95253552ab..1c5d80a8de 100644 --- a/src/LinqTests/Acceptance/Support/LinqTestContext.cs +++ b/src/LinqTests/Acceptance/Support/LinqTestContext.cs @@ -65,7 +65,7 @@ protected static void selectInOrder(Func, IQueryable> s testCases.Add(comparison); } - private static readonly string[] _methodNames = new string[] { "@where", nameof(ordered), nameof(unordered), nameof(selectInOrder) }; + private static readonly string[] _methodNames = new string[] { "@where", nameof(ordered), nameof(unordered), nameof(selectInOrder), "select" }; private static readonly string[] _descriptions; protected static string[] readDescriptions() diff --git a/src/LinqTests/Acceptance/Support/SelectTransform.cs b/src/LinqTests/Acceptance/Support/SelectTransform.cs new file mode 100644 index 0000000000..fd264c87df --- /dev/null +++ b/src/LinqTests/Acceptance/Support/SelectTransform.cs @@ -0,0 +1,40 @@ +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using FastExpressionCompiler; +using Marten; +using Marten.Testing.Documents; +using Marten.Testing.Harness; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Shouldly; + +namespace LinqTests.Acceptance.Support; + +public class SelectTransform: LinqTestCase +{ + private readonly Expression> _selector; + + public SelectTransform(Expression> selector) + { + _selector = selector; + } + + public override async Task Compare(IQuerySession session, Target[] documents, TestOutputMartenLogger logger) + { + var target = documents.FirstOrDefault(x => x.StringArray?.Length > 0 && x.NumberArray?.Length > 0 && x.Inner != null); + var expected = documents.Select(_selector.CompileFast()).Take(1).Single(); + + var actual = await session.Query().Where(x => x.Id == target.Id).Select(_selector).SingleAsync(); + + var expectedJson = JsonConvert.SerializeObject(expected); + var actualJson = JsonConvert.SerializeObject(actual); + + if (!JToken.DeepEquals(JObject.Parse(expectedJson), JObject.Parse(actualJson))) + { + // This would you would assume throw + actualJson.ShouldBe(expectedJson); + } + } +} diff --git a/src/LinqTests/Acceptance/select_clause_usage.cs b/src/LinqTests/Acceptance/select_clause_usage.cs index d9378cba0d..d2bfdc5144 100644 --- a/src/LinqTests/Acceptance/select_clause_usage.cs +++ b/src/LinqTests/Acceptance/select_clause_usage.cs @@ -44,6 +44,25 @@ public void use_select_in_query_for_one_field_and_first() .First().ShouldBe("Bill"); } + [Fact] + public async Task use_select_in_query_for_first_in_collection() + { + theSession.Store(new User { FirstName = "Hank", Roles = new []{"a", "b", "c"}}); + theSession.Store(new User { FirstName = "Bill", Roles = new []{"d", "b", "c"} }); + theSession.Store(new User { FirstName = "Sam", Roles = new []{"e", "b", "c"} }); + theSession.Store(new User { FirstName = "Tom", Roles = new []{"f", "b", "c"} }); + + await theSession.SaveChangesAsync(); + + var data = await theSession + .Query() + .OrderBy(x => x.FirstName) + .Select(x => new { Id = x.Id, Role = x.Roles[0] }) + .ToListAsync(); + + data[0].Role.ShouldBe("d"); + } + [Fact] public async Task use_select_in_query_for_one_field_async() { diff --git a/src/LinqTests/Acceptance/select_clauses.cs b/src/LinqTests/Acceptance/select_clauses.cs new file mode 100644 index 0000000000..f9b682b37f --- /dev/null +++ b/src/LinqTests/Acceptance/select_clauses.cs @@ -0,0 +1,85 @@ +using System; +using System.Linq.Expressions; +using System.Threading.Tasks; +using LinqTests.Acceptance.Support; +using Marten.Testing.Documents; +using Xunit.Abstractions; + +namespace LinqTests.Acceptance; + +public class select_clauses : LinqTestContext +{ + public select_clauses(DefaultQueryFixture fixture, ITestOutputHelper output) : base(fixture) + { + TestOutput = output; + } + + private static void select(Expression> selector) + { + testCases.Add(new SelectTransform(selector)); + } + + static select_clauses() + { + var number = 10; + + select(x => new {Id = x.Id}); + select(x => new {Foo = x.Id}); + select(x => new {Id = x.Id, Inner = x.Inner}); + select(x => new {Id = x.Id, Number = x.Number}); + select(x => new {Id = x.Id, Other = x.NumberArray}); + select(x => new {Id = x.Id, Other = x.Color}); + select(x => new {Id = x.Id, Other = x.Children}); + select(x => new {Id = x.Id, Other = x.Date}); + select(x => new {Id = x.Id, Other = x.Decimal}); + select(x => new {Id = x.Id, Other = x.Double}); + select(x => new {Id = x.Id, Other = x.Flag}); + select(x => new {Id = x.Id, Other = x.Double}); + select(x => new {Id = x.Id, Other = x.Long}); + select(x => new {Id = x.Id, Other = x.DateOffset}); + select(x => new {Id = x.Id, Other = x.GuidArray}); + select(x => new {Id = x.Id, Other = x.GuidDict}); + select(x => new {Id = x.Id, Other = x.Float}); + select(x => new {Id = x.Id, Other = x.NullableBoolean}); + select(x => new {Id = x.Id, Other = x.NullableColor}); + select(x => new {Id = x.Id, Other = x.StringArray}); + select(x => new {Id = x.Id, Other = x.StringDict}); + select(x => new {Id = x.Id, Other = x.TagsHashSet}); + select(x => new {Id = x.Id, Name = x.String}); + select(x => new {Id = x.Id, Name = "Harold"}); + select(x => new {Id = x.Inner.Number, Name = x.Inner.String}); + select(x => new {Id = 5, Name = x.Inner.String}); + select(x => new {Id = number, Name = x.Inner.String}); + select(x => new { Id = x.Id, Name = x.StringArray[0] }); + select(x => new { Id = x.Id, Age = x.NumberArray[0] }); + + select(x => new Person { Age = x.Number, Name = x.String }); + select(x => new Person(x.String, x.Number)); + + select(x => new { Id = x.Id, Person = new Person { Age = x.Number, Name = x.String } }); + select(x => new { Id = x.Id, Person = new Person(x.String, x.Number) }); + } + + [Theory] + [MemberData(nameof(GetDescriptions))] + public Task run_query(string description) + { + return assertTestCase(description, Fixture.Store); + } + + public class Person + { + public Person() + { + } + + public Person(string name, int age) + { + Name = name; + Age = age; + } + + public string Name { get; set; } + public int Age { get; set; } + } +} diff --git a/src/Marten/Linq/Members/ValueCollections/ValueCollectionMember.cs b/src/Marten/Linq/Members/ValueCollections/ValueCollectionMember.cs index b9bbf84833..c6b954839a 100644 --- a/src/Marten/Linq/Members/ValueCollections/ValueCollectionMember.cs +++ b/src/Marten/Linq/Members/ValueCollections/ValueCollectionMember.cs @@ -17,7 +17,7 @@ namespace Marten.Linq.Members.ValueCollections; -internal class ValueCollectionMember: QueryableMember, ICollectionMember, IValueCollectionMember +internal class ValueCollectionMember: QueryableMember, ICollectionMember, IValueCollectionMember, ISelectableMember { private readonly IQueryableMember _count; private readonly WholeDataMember _wholeDataMember; @@ -194,4 +194,9 @@ public IEnumerator GetEnumerator() { throw new NotSupportedException(); } + + public void Apply(CommandBuilder builder, ISerializer serializer) + { + builder.Append(RawLocator); + } } diff --git a/src/Marten/Linq/Parsing/SelectParser.cs b/src/Marten/Linq/Parsing/SelectParser.cs new file mode 100644 index 0000000000..7ae3da1b93 --- /dev/null +++ b/src/Marten/Linq/Parsing/SelectParser.cs @@ -0,0 +1,222 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using Marten.Exceptions; +using Marten.Linq.Members; +using Marten.Linq.Members.ValueCollections; +using Marten.Linq.SqlGeneration; +using Marten.Util; +using Weasel.Postgresql; +using Weasel.Postgresql.SqlGeneration; +using BindingFlags = System.Reflection.BindingFlags; + +namespace Marten.Linq.Parsing; + +internal class SelectParser: ExpressionVisitor +{ + private readonly ISerializer _serializer; + private readonly IQueryableMemberCollection _members; + private string _currentField; + + public SelectParser(ISerializer serializer, IQueryableMemberCollection members, Expression expression) + { + NewObject = new NewObject(serializer); + _serializer = serializer; + _members = members; + Visit(expression); + } + + public NewObject NewObject { get; private set; } + + public override Expression Visit(Expression node) + { + return base.Visit(node); + } + + protected override Expression VisitBinary(BinaryExpression node) + { + if (node.TryToParseConstant(out var constant)) + { + VisitConstant(constant); + return null; + } + + switch (node.NodeType) + { + case ExpressionType.ArrayIndex: + var index = (int)node.Right.ReduceToConstant().Value; + + var inner = _members.MemberFor(node.Left); + if (inner is IHasChildrenMembers parent) + { + var member = parent.FindMember(new ArrayIndexMember(index)); + NewObject.Members[_currentField] = member; + + _currentField = null; + } + else + { + throw new BadLinqExpressionException("Marten is not (yet) able to process this Select() transform"); + } + + return null; + } + + return base.VisitBinary(node); + } + + protected override Expression VisitConstant(ConstantExpression node) + { + var value = node.ReduceToConstant(); + var raw = value.Value; + if (raw is string r) + { + NewObject.Members[_currentField] = new LiteralSql($"'{r.TrimStart('"').TrimEnd('"')}'"); + } + else if (raw is null) + { + NewObject.Members[_currentField] = new LiteralSql("''"); + } + else + { + NewObject.Members[_currentField] = new LiteralSql(raw.ToString()); + } + + _currentField = null; + + return base.VisitConstant(node); + } + + protected override Expression VisitMemberInit(MemberInitExpression node) + { + var child = new SelectParser(_serializer, _members, node.NewExpression); + foreach (var binding in node.Bindings.OfType()) + { + child.ReadBinding(binding); + } + + if (_currentField == null) + { + // It's from an x => new Person{Age = x.Number, Name = x.Name} kind + // of transform, so use the child's new object + NewObject = child.NewObject; + } + else + { + NewObject.Members[_currentField] = child.NewObject; + } + + return null; + } + + public void ReadBinding(MemberAssignment binding) + { + _currentField = binding.Member.Name; + Visit(binding.Expression); + } + + protected override MemberBinding VisitMemberBinding(MemberBinding node) + { + _currentField = node.Member.Name; + + return base.VisitMemberBinding(node); + } + + protected override Expression VisitMember(MemberExpression node) + { + if (_currentField == null) return base.VisitMember(node); + + if (node.TryToParseConstant(out var constant)) + { + VisitConstant(constant); + return null; + } + + var member = _members.MemberFor(node); + NewObject.Members[_currentField] = member; + _currentField = null; + + return base.VisitMember(node); + } + + private bool _hasStarted; + + protected override Expression VisitNew(NewExpression node) + { + if (_hasStarted) + { + var child = new SelectParser(_serializer, _members, node); + NewObject.Members[_currentField] = child.NewObject; + + return null; + } + + _hasStarted = true; + + var parameters = node.Constructor.GetParameters(); + + for (var i = 0; i < parameters.Length; i++) + { + _currentField = parameters[i].Name; + Visit(node.Arguments[i]); + } + + return node; + } + +} + + +public interface ISelectableMember +{ + void Apply(CommandBuilder builder, ISerializer serializer); +} + +internal class NewObject : ISqlFragment +{ + private readonly ISerializer _serializer; + + public NewObject(ISerializer serializer) + { + _serializer = serializer; + } + + public Dictionary Members { get; } = new(); + + public void Apply(CommandBuilder builder) + { + builder.Append(" jsonb_build_object("); + + var pairs = Members.ToArray(); + for (int i = 0; i < pairs.Length - 1; i++) + { + writeMember(builder, pairs[i]); + builder.Append(", "); + } + + writeMember(builder, pairs.Last()); + + builder.Append(") "); + } + + private void writeMember(CommandBuilder builder, KeyValuePair pair) + { + builder.Append($"'{pair.Key.FormatCase(_serializer.Casing)}', "); + if (pair.Value is ISelectableMember selectable) + { + selectable.Apply(builder, _serializer); + } + else + { + pair.Value.Apply(builder); + } + } + + public bool Contains(string sqlText) + { + return false; + } +} diff --git a/src/Marten/Linq/Parsing/SelectTransformBuilder.cs b/src/Marten/Linq/Parsing/SelectTransformBuilder.cs deleted file mode 100644 index e945ae53e7..0000000000 --- a/src/Marten/Linq/Parsing/SelectTransformBuilder.cs +++ /dev/null @@ -1,141 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; -using JasperFx.Core; -using Marten.Linq.Members; -using Marten.Linq.Members.Dictionaries; -using Marten.Util; - -namespace Marten.Linq.Parsing; - -internal class SelectTransformBuilder: ExpressionVisitor -{ - private SelectedField _currentField; - private TargetObject _target; - - public SelectTransformBuilder(Expression clause, IQueryableMemberCollection members, ISerializer serializer) - { - // ReSharper disable once VirtualMemberCallInConstructor - Visit(clause); - SelectedFieldExpression = _target.ToSelectField(members, serializer); - } - - public string SelectedFieldExpression { get; } - - protected override Expression VisitNew(NewExpression expression) - { - if (_target != null) - { - return base.VisitNew(expression); - } - - _target = new TargetObject(expression.Type); - - var parameters = expression.Constructor.GetParameters(); - - for (var i = 0; i < parameters.Length; i++) - { - _currentField = _target.StartBinding(parameters[i].Name); - Visit(expression.Arguments[i]); - } - - return expression; - } - - protected override Expression VisitMember(MemberExpression node) - { - _currentField.Add(node.Member); - return base.VisitMember(node); - } - - protected override MemberBinding VisitMemberBinding(MemberBinding node) - { - _currentField = _target.StartBinding(node.Member.Name); - - return base.VisitMemberBinding(node); - } - - - public class TargetObject - { - private readonly IList _setters = new List(); - - public TargetObject(Type type) - { - Type = type; - } - - public Type Type { get; } - - public SelectedField StartBinding(string bindingName) - { - var setter = new SetterBinding(bindingName); - _setters.Add(setter); - - return setter.Field; - } - - public string ToSelectField(IQueryableMemberCollection fields, ISerializer serializer) - { - var jsonBuildObjectArgs = _setters.Select(x => x.ToJsonBuildObjectPair(fields, serializer)).Join(", "); - return $"jsonb_build_object({jsonBuildObjectArgs})"; - } - - private class SetterBinding - { - public SetterBinding(string name) - { - Name = name; - } - - private string Name { get; } - public SelectedField Field { get; } = new(); - - public string ToJsonBuildObjectPair(IQueryableMemberCollection mapping, ISerializer serializer) - { - var field = mapping.MemberFor(Field.ToArray()); - var locator = serializer.ValueCasting == ValueCasting.Relaxed - ? field.RawLocator ?? field.TypedLocator - : field.TypedLocator; - - if (field is IDictionaryMember) - { - // DictionaryField.RawLocator does not have cast to JSONB so TypedLocator is used - locator = field.TypedLocator; - } - - if (field.MemberType.IsClass && field.MemberType != typeof(string) && - field.MemberType != typeof(decimal)) - { - // If the field is a class, we need to cast it to JSONB otherwise it will be serialized to plain string and fail to deserialize later on - locator = field.JSONBLocator; - } - - return $"'{Name.FormatCase(serializer.Casing)}', {locator}"; - } - } - } - - public class SelectedField: IEnumerable - { - private readonly Stack _members = new(); - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - public IEnumerator GetEnumerator() - { - return _members.GetEnumerator(); - } - - public void Add(MemberInfo member) - { - _members.Push(member); - } - } -} diff --git a/src/Marten/Linq/Parsing/SelectorVisitor.cs b/src/Marten/Linq/Parsing/SelectorVisitor.cs index 854d4213e3..e7bc51cac0 100644 --- a/src/Marten/Linq/Parsing/SelectorVisitor.cs +++ b/src/Marten/Linq/Parsing/SelectorVisitor.cs @@ -19,6 +19,11 @@ public SelectorVisitor(SelectorStatement statement, IQueryableMemberCollection c _serializer = serializer; } + public override Expression Visit(Expression node) + { + return base.Visit(node); + } + protected override Expression VisitUnary(UnaryExpression node) { ToScalar(node); @@ -72,12 +77,11 @@ public void ToScalar(Expression selectClauseSelector) public void ToSelectTransform(Expression selectExpression, ISerializer serializer) { - var builder = new SelectTransformBuilder(selectExpression, _collection, serializer); - var transformField = builder.SelectedFieldExpression; + var visitor = new SelectParser(_serializer, _collection, selectExpression); _statement.SelectClause = - typeof(DataSelectClause<>).CloseAndBuildAs(_statement.SelectClause.FromObject, - transformField, + typeof(SelectDataSelectClause<>).CloseAndBuildAs(_statement.SelectClause.FromObject, + visitor.NewObject, selectExpression.Type); } } diff --git a/src/Marten/Linq/SqlGeneration/Literal.cs b/src/Marten/Linq/SqlGeneration/Literal.cs new file mode 100644 index 0000000000..3f213d0bb0 --- /dev/null +++ b/src/Marten/Linq/SqlGeneration/Literal.cs @@ -0,0 +1,23 @@ +using Weasel.Postgresql; +using Weasel.Postgresql.SqlGeneration; + +namespace Marten.Linq.SqlGeneration; + +/// +/// Exactly what it sounds like, represents a little bit +/// of literal SQL within a bigger statement +/// +/// +// TODO -- move this to Weasel itself +public record LiteralSql(string Text) : ISqlFragment +{ + public void Apply(CommandBuilder builder) + { + builder.Append(Text); + } + + public bool Contains(string sqlText) + { + return Text.Contains(sqlText); + } +} diff --git a/src/Marten/Linq/SqlGeneration/SelectDataSelectClause.cs b/src/Marten/Linq/SqlGeneration/SelectDataSelectClause.cs new file mode 100644 index 0000000000..a964232b3b --- /dev/null +++ b/src/Marten/Linq/SqlGeneration/SelectDataSelectClause.cs @@ -0,0 +1,99 @@ +using System; +using JasperFx.Core; +using Marten.Internal; +using Marten.Linq.Parsing; +using Marten.Linq.QueryHandlers; +using Marten.Linq.Selectors; +using Weasel.Postgresql; +using Weasel.Postgresql.SqlGeneration; + +namespace Marten.Linq.SqlGeneration; + +internal class SelectDataSelectClause: ISelectClause, IScalarSelectClause +{ + public SelectDataSelectClause(string from, ISqlFragment selector) + { + FromObject = from; + Selector = selector; + } + + public ISqlFragment Selector { get; } + + bool ISqlFragment.Contains(string sqlText) + { + return false; + } + + public Type SelectedType => typeof(T); + + public string FromObject { get; } + + public void Apply(CommandBuilder sql) + { + sql.Append("select "); + + if (Operator.IsNotEmpty()) + { + sql.Append(Operator); + sql.Append("("); + Selector.Apply(sql); + sql.Append(")"); + } + else + { + Selector.Apply(sql); + } + + sql.Append(" as data from "); + + sql.Append(FromObject); + sql.Append(" as d"); + } + + public string[] SelectFields() + { + return new[] { "data" }; + } + + public ISelector BuildSelector(IMartenSession session) + { + return new SerializationSelector(session.Serializer); + } + + public IQueryHandler BuildHandler(IMartenSession session, ISqlFragment statement, + ISqlFragment currentStatement) + { + var selector = new SerializationSelector(session.Serializer); + + return LinqQueryParser.BuildHandler(selector, statement); + } + + public ISelectClause UseStatistics(QueryStatistics statistics) + { + return new StatsSelectClause(this, statistics); + } + + public override string ToString() + { + return $"Data from {FromObject}"; + } + + public string MemberName => "calculated"; + public void ApplyOperator(string op) + { + Operator = op; + } + + public string? Operator { get; set; } + + public ISelectClause CloneToDouble() + { + return new SelectDataSelectClause(FromObject, Selector); + } + + public ISelectClause CloneToOtherTable(string tableName) + { + return new SelectDataSelectClause(tableName, Selector); + } +} +