From 1d5db8681067996ca8def615278640a74236f304 Mon Sep 17 00:00:00 2001 From: Chris Cameron Date: Tue, 5 Oct 2021 03:07:26 -0400 Subject: [PATCH] feat: added IsMeasurement option to Column attribute (#240) --- CHANGELOG.md | 2 +- Client.Core.Test/AbstractTest.cs | 6 +- Client.Core/Attributes.cs | 4 ++ Client.Core/Flux/Internal/AttributesCache.cs | 2 +- Client.Core/Flux/Internal/FluxResultMapper.cs | 4 ++ Client.Linq.Test/DomainObjects.cs | 12 ++++ Client.Linq.Test/InfluxDBQueryVisitorTest.cs | 30 ++++++++++ Client.Linq/IMemberNameResolver.cs | 6 ++ Client.Linq/InfluxDBQueryable.cs | 37 +++++++++++- .../Internal/Expressions/ColumnName.cs | 3 + .../Expressions/MeasurementColumnName.cs | 22 +++++++ Client.Linq/Internal/QueryAggregator.cs | 41 +++++++++++-- .../Internal/QueryExpressionTreeVisitor.cs | 3 + Client.Linq/Internal/QueryVisitor.cs | 5 +- Client.Linq/README.md | 56 +++++++++++++++++- Client.Test/ItTasksApiTest.cs | 6 +- Client.Test/MeasurementMapperTest.cs | 59 ++++++++++++++++++- Client.Test/QueryApiTest.cs | 3 + Client/Internal/MeasurementMapper.cs | 20 +++++-- 19 files changed, 299 insertions(+), 22 deletions(-) create mode 100644 Client.Linq/Internal/Expressions/MeasurementColumnName.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a7570af3..ce81df14b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features 1. [#239](https://github.com/influxdata/influxdb-client-csharp/pull/239): Add support for Asynchronous queries [LINQ] +1. [#240](https://github.com/influxdata/influxdb-client-csharp/pull/240): Add IsMeasurement option to Column attribute for dynamic measurement names in POCO classes ## 3.0.0 [2021-09-17] @@ -11,7 +12,6 @@ Adds a `Type` overload for POCOs to `QueryAsync`. This will add `object ConvertT ### Features 1. [#232](https://github.com/influxdata/influxdb-client-csharp/pull/232): Add a `Type` overload for POCOs to `QueryAsync`. 1. [#233](https://github.com/influxdata/influxdb-client-csharp/pull/233): Add possibility to follow HTTP redirects -1. [#239](https://github.com/influxdata/influxdb-client-csharp/pull/239): Add support for Asynchronous queries [LINQ] ### Bug Fixes 1. [#236](https://github.com/influxdata/influxdb-client-csharp/pull/236): Mapping `long` type into Flux AST [LINQ] diff --git a/Client.Core.Test/AbstractTest.cs b/Client.Core.Test/AbstractTest.cs index eb84addb4..af6316c51 100644 --- a/Client.Core.Test/AbstractTest.cs +++ b/Client.Core.Test/AbstractTest.cs @@ -114,13 +114,13 @@ private async Task InfluxDbRequest(HttpRequestMessage request) try { var response = await httpClient.SendAsync(request); - Assert.IsTrue(response.IsSuccessStatusCode); + Assert.IsTrue(response.IsSuccessStatusCode, $"Failed to make HTTP request: {response.ReasonPhrase}"); Thread.Sleep(DefaultInfluxDBSleep); } - catch (Exception) + catch (Exception e) { - Assert.Fail("Unexpected exception"); + Assert.Fail("Unexpected exception: " + e); } } } diff --git a/Client.Core/Attributes.cs b/Client.Core/Attributes.cs index 9d2976779..6575d6f50 100644 --- a/Client.Core/Attributes.cs +++ b/Client.Core/Attributes.cs @@ -5,6 +5,7 @@ namespace InfluxDB.Client.Core /// /// The annotation is used for mapping POCO class into line protocol. /// + [AttributeUsage(AttributeTargets.Class)] public sealed class Measurement : Attribute { public string Name { get; } @@ -18,10 +19,13 @@ public Measurement(string name) /// /// The annotation is used to customize bidirectional mapping between POCO and Flux query result or Line Protocol. /// + [AttributeUsage(AttributeTargets.Property)] public sealed class Column : Attribute { public string Name { get; } + public bool IsMeasurement { get; set; } + public bool IsTag { get; set; } public bool IsTimestamp { get; set; } diff --git a/Client.Core/Flux/Internal/AttributesCache.cs b/Client.Core/Flux/Internal/AttributesCache.cs index 91cda2501..7b9681451 100644 --- a/Client.Core/Flux/Internal/AttributesCache.cs +++ b/Client.Core/Flux/Internal/AttributesCache.cs @@ -30,7 +30,7 @@ public PropertyInfo[] GetProperties(Type type) } /// - /// Get Mapping attribute for specified propery. + /// Get Mapping attribute for specified property. /// /// property of DomainObject /// Property Attribute diff --git a/Client.Core/Flux/Internal/FluxResultMapper.cs b/Client.Core/Flux/Internal/FluxResultMapper.cs index 180c27946..1fd817ad7 100644 --- a/Client.Core/Flux/Internal/FluxResultMapper.cs +++ b/Client.Core/Flux/Internal/FluxResultMapper.cs @@ -69,6 +69,10 @@ internal object ToPoco(FluxRecord record, Type type) { var attribute = _attributesCache.GetAttribute(property); + if (attribute != null && attribute.IsMeasurement) + { + SetFieldValue(poco, property, record.GetMeasurement()); + } if (attribute != null && attribute.IsTimestamp) { SetFieldValue(poco, property, record.GetTime()); diff --git a/Client.Linq.Test/DomainObjects.cs b/Client.Linq.Test/DomainObjects.cs index af1a51c0c..9228c5db2 100644 --- a/Client.Linq.Test/DomainObjects.cs +++ b/Client.Linq.Test/DomainObjects.cs @@ -37,6 +37,18 @@ class SensorDateTimeOffset [Column(IsTimestamp = true)] public DateTimeOffset Timestamp { get; set; } } + class SensorWithCustomMeasurement + { + [Column(IsMeasurement = true)] + public string Measurement { get; set; } + + [Column("sensor_id", IsTag = true)] + public string SensorId { get; set; } + + [Column("data")] + public int Value { get; set; } + } + class SensorCustom { public Guid Id { get; set; } diff --git a/Client.Linq.Test/InfluxDBQueryVisitorTest.cs b/Client.Linq.Test/InfluxDBQueryVisitorTest.cs index a1516675f..6760282d6 100644 --- a/Client.Linq.Test/InfluxDBQueryVisitorTest.cs +++ b/Client.Linq.Test/InfluxDBQueryVisitorTest.cs @@ -428,6 +428,36 @@ where month11 > s.Timestamp } } + [Test] + public void ResultOperatorByMeasurement() + { + var settings = new QueryableOptimizerSettings + { + DropMeasurementColumn = false + }; + + var query = from s in InfluxDBQueryable.Queryable("my-bucket", "my-org", _queryApi, settings) + where s.Value > 10 + where s.Measurement == "my-measurement" + select s; + var visitor = BuildQueryVisitor(query); + + const string expected = "start_shifted = int(v: time(v: p2))\n\nfrom(bucket: p1) " + + "|> range(start: time(v: start_shifted)) " + + "|> filter(fn: (r) => (r[\"_measurement\"] == p4)) " + + "|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\") " + + "|> drop(columns: [\"_start\", \"_stop\"]) " + + "|> filter(fn: (r) => (r[\"data\"] > p3))"; + + Assert.AreEqual(expected, visitor.BuildFluxQuery()); + + var ast = visitor.BuildFluxAST(); + + var measurementAssignment = ((OptionStatement)ast.Body[3]).Assignment as VariableAssignment; + Assert.AreEqual("p4", measurementAssignment?.Id.Name); + Assert.AreEqual("my-measurement", (measurementAssignment?.Init as StringLiteral)?.Value); + } + [Test] public void TimestampAsDateTimeOffset() { diff --git a/Client.Linq/IMemberNameResolver.cs b/Client.Linq/IMemberNameResolver.cs index fbae526e1..6fb4e60c2 100644 --- a/Client.Linq/IMemberNameResolver.cs +++ b/Client.Linq/IMemberNameResolver.cs @@ -33,6 +33,7 @@ public interface IMemberNameResolver public enum MemberType { + Measurement, Tag, Field, Timestamp, @@ -52,6 +53,11 @@ public MemberType ResolveMemberType(MemberInfo memberInfo) if (attribute != null) { + if (attribute.IsMeasurement) + { + return MemberType.Measurement; + } + if (attribute.IsTag) { return MemberType.Tag; diff --git a/Client.Linq/InfluxDBQueryable.cs b/Client.Linq/InfluxDBQueryable.cs index c4d604be6..5824c86d1 100644 --- a/Client.Linq/InfluxDBQueryable.cs +++ b/Client.Linq/InfluxDBQueryable.cs @@ -18,17 +18,50 @@ public class QueryableOptimizerSettings public QueryableOptimizerSettings() { QueryMultipleTimeSeries = false; + DropMeasurementColumn = true; + DropStartColumn = true; + DropStopColumn = true; } /// /// Gets or sets whether the drive is used to query multiple time series. /// Setting this variable to true will change how the produced Flux Query looks like: /// - /// appends group operator - /// enable use default sorting: sort(columns: ["_time"], desc: false) + /// Appends group operator + /// Enable use default sorting: sort(columns: ["_time"], desc: false) /// /// public bool QueryMultipleTimeSeries { get; set; } + + /// + /// Gets or sets whether the _measurement column will be dropped from query results. + /// Setting this variable to true will change how the produced Flux Query looks like: + /// + /// Appends drop operator + /// Drops the _measurement column: drop(columns: ["_measurement"]) + /// + /// + public bool DropMeasurementColumn { get; set; } + + /// + /// Gets or sets whether the _start column will be dropped from query results. + /// Setting this variable to true will change how the produced Flux Query looks like: + /// + /// Appends drop operator + /// Drops the _start column: drop(columns: ["_start"]) + /// + /// + public bool DropStartColumn { get; set; } + + /// + /// Gets or sets whether the _stop column will be dropped from query results. + /// Setting this variable to true will change how the produced Flux Query looks like: + /// + /// Appends drop operator + /// Drops the _stop column: drop(columns: ["_stop"]) + /// + /// + public bool DropStopColumn { get; set; } } /// diff --git a/Client.Linq/Internal/Expressions/ColumnName.cs b/Client.Linq/Internal/Expressions/ColumnName.cs index 7c07c0769..5e1eea0f9 100644 --- a/Client.Linq/Internal/Expressions/ColumnName.cs +++ b/Client.Linq/Internal/Expressions/ColumnName.cs @@ -18,6 +18,9 @@ public void AppendFlux(StringBuilder builder) { switch (_memberResolver.ResolveMemberType(_member)) { + case MemberType.Measurement: + builder.Append("_measurement"); + break; case MemberType.Timestamp: builder.Append("_time"); break; diff --git a/Client.Linq/Internal/Expressions/MeasurementColumnName.cs b/Client.Linq/Internal/Expressions/MeasurementColumnName.cs new file mode 100644 index 000000000..d53742110 --- /dev/null +++ b/Client.Linq/Internal/Expressions/MeasurementColumnName.cs @@ -0,0 +1,22 @@ +using System.Reflection; +using System.Text; + +namespace InfluxDB.Client.Linq.Internal.Expressions +{ + internal class MeasurementColumnName : IExpressionPart + { + private readonly ColumnName _delegate; + + internal MeasurementColumnName(MemberInfo member, IMemberNameResolver memberNameResolver) + { + _delegate = new ColumnName(member, memberNameResolver); + } + + public void AppendFlux(StringBuilder builder) + { + builder.Append("r[\""); + _delegate.AppendFlux(builder); + builder.Append("\"]"); + } + } +} \ No newline at end of file diff --git a/Client.Linq/Internal/QueryAggregator.cs b/Client.Linq/Internal/QueryAggregator.cs index e20a20595..98a83d76a 100644 --- a/Client.Linq/Internal/QueryAggregator.cs +++ b/Client.Linq/Internal/QueryAggregator.cs @@ -155,12 +155,18 @@ internal string BuildFluxQuery(QueryableOptimizerSettings settings) BuildOperator("from", "bucket", _bucketAssignment), BuildRange(transforms), BuildFilter(_filterByTags), - "pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")", - "drop(columns: [\"_start\", \"_stop\", \"_measurement\"])", - settings.QueryMultipleTimeSeries ? "group()" : "", - BuildFilter(_filterByFields) + "pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")" }; + var drop = BuildDrop(settings); + if (!string.IsNullOrEmpty(drop)) + { + parts.Add(drop); + } + + parts.Add(settings.QueryMultipleTimeSeries ? "group()" : ""); + parts.Add(BuildFilter(_filterByFields)); + // https://docs.influxdata.com/influxdb/cloud/reference/flux/stdlib/built-in/transformations/sort/ foreach (var ((column, columnVariable, descending, descendingVariable), index) in _orders.Select((value, i) => (value, i))) { @@ -203,6 +209,33 @@ internal string BuildFluxQuery(QueryableOptimizerSettings settings) return query.ToString(); } + private string BuildDrop(QueryableOptimizerSettings settings) + { + var columns = new List(); + + if (settings.DropStartColumn) + { + columns.Add("\"_start\""); + } + + if (settings.DropStopColumn) + { + columns.Add("\"_stop\""); + } + + if (settings.DropMeasurementColumn) + { + columns.Add("\"_measurement\""); + } + + if (columns.Count == 0) + { + return null; + } + + return $"drop(columns: [{string.Join(", ", columns)}])"; + } + private string BuildRange(List transforms) { string rangeStartShift = null; diff --git a/Client.Linq/Internal/QueryExpressionTreeVisitor.cs b/Client.Linq/Internal/QueryExpressionTreeVisitor.cs index 94232d37b..72c2c90d8 100644 --- a/Client.Linq/Internal/QueryExpressionTreeVisitor.cs +++ b/Client.Linq/Internal/QueryExpressionTreeVisitor.cs @@ -102,6 +102,9 @@ protected override Expression VisitMember(MemberExpression expression) { switch (_context.MemberResolver.ResolveMemberType(expression.Member)) { + case MemberType.Measurement: + _expressionParts.Add(new MeasurementColumnName(expression.Member, _context.MemberResolver)); + break; case MemberType.Timestamp: _expressionParts.Add(new TimeColumnName(expression.Member, _context.MemberResolver)); break; diff --git a/Client.Linq/Internal/QueryVisitor.cs b/Client.Linq/Internal/QueryVisitor.cs index 6b6421210..b6b1fd2d5 100644 --- a/Client.Linq/Internal/QueryVisitor.cs +++ b/Client.Linq/Internal/QueryVisitor.cs @@ -75,7 +75,7 @@ public override void VisitWhereClause(WhereClause whereClause, QueryModel queryM var tagFilter = new List(); var fieldFilter = new List(); - // Map LINQ filter expresion to right place: range, tag filtering, field filtering + // Map LINQ filter expression to right place: range, tag filtering, field filtering foreach (var expression in expressions) { switch (expression) @@ -86,6 +86,7 @@ public override void VisitWhereClause(WhereClause whereClause, QueryModel queryM break; // Tag case TagColumnName _: + case MeasurementColumnName _: tagFilter.Add(expression); break; // Field @@ -187,7 +188,7 @@ private string ConcatExpression(IEnumerable expressions) }).ToString(); } - private void AddFilterByRange(List rangeFilter) + private void AddFilterByRange(List rangeFilter) { var rangeBinaryIndexes = Enumerable.Range(0, rangeFilter.Count) .Where(i => rangeFilter[i] is BinaryOperator) diff --git a/Client.Linq/README.md b/Client.Linq/README.md index ba5e99c94..2d4e30417 100644 --- a/Client.Linq/README.md +++ b/Client.Linq/README.md @@ -38,6 +38,7 @@ This section contains links to the client library documentation. - [LongCount](#longcount) - [Domain Converter](#domain-converter) - [How to debug output Flux Query](#how-to-debug-output-flux-query) +- [How to filter by Measurement](#how-to-filter-by-measurement) - [Asynchronous Queries](#asynchronous-queries) ## How to start @@ -997,6 +998,59 @@ Console.WriteLine(); Console.WriteLine(influxQuery._Query); ``` +## How to filter by Measurement + +By default, as an optimization step, ***Flux queries generated by LINQ will automatically drop the Start, Stop and Measurement columns***: + +```flux +from(bucket: "my-bucket") + |> range(start: 0) + |> drop(columns: ["_start", "_stop", "_measurement"]) + ... +``` +This is because typical POCO classes do not include them: +```c# +[Measurement("temperature")] +private class Temperature +{ + [Column("location", IsTag = true)] public string Location { get; set; } + [Column("value")] public double Value { get; set; } + [Column(IsTimestamp = true)] public DateTime Time { get; set; } +} +``` + +It is, however, possible to utilize the Measurement column in LINQ queries by enabling it in the query optimization settings: + +```c# +var optimizerSettings = + new QueryableOptimizerSettings + { + DropMeasurementColumn = false, + + // Note we can also enable the start and stop columns + //DropStartColumn = false, + //DropStopColumn = false + }; + +var queryable = + new InfluxDBQueryable("my-bucket", "my-org", queryApi, new DefaultMemberNameResolver(), optimizerSettings); + +var latest = + await queryable.Where(p => p.Measurement == "temperature") + .OrderByDescending(p => p.Time) + .ToInfluxQueryable() + .GetAsyncEnumerator() + .FirstOrDefaultAsync(); + +private class InfluxPoint +{ + [Column(IsMeasurement = true)] public string Measurement { get; set; } + [Column("location", IsTag = true)] public string Location { get; set; } + [Column("value")] public double Value { get; set; } + [Column(IsTimestamp = true)] public DateTime Time { get; set; } +} +``` + ## Asynchronous Queries The LINQ driver also supports asynchronous querying. For asynchronous queries you have to initialize `InfluxDBQueryable` with asynchronous version of [QueryApi](/Client/QueryApi.cs) and transform `IQueryable` to `IAsyncEnumerable`: @@ -1011,4 +1065,4 @@ var query = from s in InfluxDBQueryable.Queryable("my-bucket", "my-org", IAsyncEnumerable enumerable = query .ToInfluxQueryable() .GetAsyncEnumerator(); -``` \ No newline at end of file +``` diff --git a/Client.Test/ItTasksApiTest.cs b/Client.Test/ItTasksApiTest.cs index 682775da2..7f5b9a0e0 100644 --- a/Client.Test/ItTasksApiTest.cs +++ b/Client.Test/ItTasksApiTest.cs @@ -527,9 +527,9 @@ public async Task Runs() Assert.IsNotEmpty(run.Id); Assert.AreEqual(task.Id, run.TaskID); Assert.AreEqual(Run.StatusEnum.Success, run.Status); - Assert.Greater(DateTime.Now, run.StartedAt); - Assert.Greater(DateTime.Now, run.FinishedAt); - Assert.Greater(DateTime.Now, run.ScheduledFor); + Assert.Greater(DateTime.UtcNow, run.StartedAt); + Assert.Greater(DateTime.UtcNow, run.FinishedAt); + Assert.Greater(DateTime.UtcNow, run.ScheduledFor); Assert.IsNull(run.RequestedAt); task = await _tasksApi.FindTaskByIdAsync(task.Id); diff --git a/Client.Test/MeasurementMapperTest.cs b/Client.Test/MeasurementMapperTest.cs index f62d4e185..93fd0d2b6 100644 --- a/Client.Test/MeasurementMapperTest.cs +++ b/Client.Test/MeasurementMapperTest.cs @@ -91,7 +91,36 @@ public void HeavyLoad() Assert.LessOrEqual(ts.Seconds, 10, $"Elapsed time: {elapsedTime}"); } - + + [Test] + public void MeasurementProperty() + { + var poco = new MeasurementPropertyPoco + { + Measurement = "poco", + Tag = "tag val", + Value = 15.444, + ValueWithoutDefaultName = 20, + ValueWithEmptyName = 25d, + Timestamp = TimeSpan.FromDays(10) + }; + + var lineProtocol = _mapper.ToPoint(poco, WritePrecision.S).ToLineProtocol(); + + Assert.AreEqual("poco,tag=tag\\ val value=15.444,ValueWithEmptyName=25,ValueWithoutDefaultName=20i 864000", lineProtocol); + } + + [Test] + public void MeasurementPropertyValidation() + { + var poco = new BadMeasurementAttributesPoco + { + Measurement = "poco" + }; + + Assert.Throws(() => _mapper.ToPoint(poco, WritePrecision.S)); + } + private class MyClass { public override string ToString() @@ -118,5 +147,33 @@ private class Poco [Column(IsTimestamp = true)] public Object Timestamp { get; set; } } + + private class MeasurementPropertyPoco + { + [Column(IsMeasurement = true)] + public string Measurement { get; set; } + + [Column("tag", IsTag = true)] + public string Tag { get; set; } + + [Column("value")] + public Object Value { get; set; } + + [Column] + public int? ValueWithoutDefaultName { get; set; } + + [Column("")] + public Double? ValueWithEmptyName { get; set; } + + [Column(IsTimestamp = true)] + public Object Timestamp { get; set; } + } + + [Measurement("poco")] + private class BadMeasurementAttributesPoco + { + [Column(IsMeasurement = true)] + public string Measurement { get; set; } + } } } \ No newline at end of file diff --git a/Client.Test/QueryApiTest.cs b/Client.Test/QueryApiTest.cs index dcdfef08c..f097c577e 100644 --- a/Client.Test/QueryApiTest.cs +++ b/Client.Test/QueryApiTest.cs @@ -79,6 +79,7 @@ public async Task GenericAndTypeofCalls() Assert.AreEqual(13.00, measurements[1].Value); Assert.IsAssignableFrom(measurementsTypeof[0]); var cast = measurementsTypeof.Cast().ToList(); + Assert.AreEqual(measurements[0].Measurement, cast[0].Measurement); Assert.AreEqual(measurements[0].Timestamp, cast[0].Timestamp); Assert.AreEqual(12.25, cast[0].Value); Assert.AreEqual(13.00, cast[1].Value); @@ -106,6 +107,8 @@ public async Task QueryAsyncEnumerable() private class SyncPoco { + [Column(IsMeasurement = true)] public string Measurement { get; set; } + [Column("id", IsTag = true)] public string Tag { get; set; } [Column("_value")] public double Value { get; set; } diff --git a/Client/Internal/MeasurementMapper.cs b/Client/Internal/MeasurementMapper.cs index 3cf70ddb1..91e594ae3 100644 --- a/Client/Internal/MeasurementMapper.cs +++ b/Client/Internal/MeasurementMapper.cs @@ -35,18 +35,30 @@ internal PointData ToPoint(TM measurement, WritePrecision precision) var measurementType = measurement.GetType(); CacheMeasurementClass(measurementType); - + var measurementAttribute = (Measurement) measurementType.GetCustomAttribute(typeof(Measurement)); - if (measurementAttribute == null) + var measurementColumn = CACHE[measurementType.Name].SingleOrDefault(p => p.Column.IsMeasurement); + + if (((measurementAttribute == null) ^ (measurementColumn == null)) == false) { throw new InvalidOperationException( - $"Measurement {measurement} does not have a {typeof(Measurement)} attribute."); + $"Unable to determine Measurement for {measurement}. Does it have a {typeof(Measurement)} or IsMessage {typeof(Column)} attribute?"); } - var point = PointData.Measurement(measurementAttribute.Name); + string measurementName = + measurementAttribute == null + ? (string)measurementColumn.Property.GetValue(measurement) + : measurementAttribute.Name; + + var point = PointData.Measurement(measurementName); foreach (var propertyInfo in CACHE[measurementType.Name]) { + if (propertyInfo.Column.IsMeasurement) + { + continue; + } + var value = propertyInfo.Property.GetValue(measurement); if (value == null) {