From 6d50fd0734aa5193a44cc08d916ae4f768a08714 Mon Sep 17 00:00:00 2001 From: mvadu Date: Sat, 3 Jun 2017 07:59:04 -0700 Subject: [PATCH] Fix #30 Treat NanoSeconds as default precision Fix #31 Partial writes raising IndexOutOfRangeException --- CHANGELOG.md | 19 +++++ .../AdysTech.InfluxDB.Client.Net.Core.csproj | 2 +- .../AdysTech.InfluxDB.Client.Net.csproj | 2 +- .../Properties/AssemblyInfo.cs | 4 +- .../packages.config | 2 +- src/DataStructures/InfluxDBClient.cs | 16 +++-- src/Extensions/EpochHelper.cs | 34 ++++----- ...sTech.InfluxDB.Client.Net.Core.Test.csproj | 4 +- tests/InfluxDBClientTest.cs | 70 +++++++++++++++++++ 9 files changed, 125 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2380081..ee5d510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## v0.6.1 [6/03/2017] + +### Release Notes +Allow points to be passed without explictly setting time or precision. + + +### Bugfixes + +- [#31](https://github.com/AdysTech/InfluxDB.Client.Net/issues/31): IndexOutOfRangeException thrown for partial writes + +### Features + +- [#30](https://github.com/AdysTech/InfluxDB.Client.Net/issues/30): Use NanoSeconds as default precision + + +### Breaking Change +- Library will silently drop points older than retention period. This is similar to InfluDB behavios where it will reject those points with an `{"error":"partial write: points beyond retention policy dropped=225"}` + + ## v0.6.1 [3/28/2017] ### Release Notes diff --git a/src/AdysTech.InfluxDB.Client.Net.Core/AdysTech.InfluxDB.Client.Net.Core.csproj b/src/AdysTech.InfluxDB.Client.Net.Core/AdysTech.InfluxDB.Client.Net.Core.csproj index 4436b44..70aad0a 100644 --- a/src/AdysTech.InfluxDB.Client.Net.Core/AdysTech.InfluxDB.Client.Net.Core.csproj +++ b/src/AdysTech.InfluxDB.Client.Net.Core/AdysTech.InfluxDB.Client.Net.Core.csproj @@ -52,7 +52,7 @@ It currently supports - + \ No newline at end of file diff --git a/src/AdysTech.InfluxDB.Client.Net/AdysTech.InfluxDB.Client.Net.csproj b/src/AdysTech.InfluxDB.Client.Net/AdysTech.InfluxDB.Client.Net.csproj index dc8a50c..aad7f19 100644 --- a/src/AdysTech.InfluxDB.Client.Net/AdysTech.InfluxDB.Client.Net.csproj +++ b/src/AdysTech.InfluxDB.Client.Net/AdysTech.InfluxDB.Client.Net.csproj @@ -32,7 +32,7 @@ - ..\..\packages\Newtonsoft.Json.10.0.1\lib\net45\Newtonsoft.Json.dll + ..\..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll diff --git a/src/AdysTech.InfluxDB.Client.Net/Properties/AssemblyInfo.cs b/src/AdysTech.InfluxDB.Client.Net/Properties/AssemblyInfo.cs index b3bae8e..c0375ea 100644 --- a/src/AdysTech.InfluxDB.Client.Net/Properties/AssemblyInfo.cs +++ b/src/AdysTech.InfluxDB.Client.Net/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.6.1.0")] -[assembly: AssemblyFileVersion("0.6.1.0")] +[assembly: AssemblyVersion("0.6.5.0")] +[assembly: AssemblyFileVersion("0.6.5.0")] diff --git a/src/AdysTech.InfluxDB.Client.Net/packages.config b/src/AdysTech.InfluxDB.Client.Net/packages.config index 3ff0027..8a2091a 100644 --- a/src/AdysTech.InfluxDB.Client.Net/packages.config +++ b/src/AdysTech.InfluxDB.Client.Net/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file diff --git a/src/DataStructures/InfluxDBClient.cs b/src/DataStructures/InfluxDBClient.cs index 2d9e17b..7fe6a7b 100644 --- a/src/DataStructures/InfluxDBClient.cs +++ b/src/DataStructures/InfluxDBClient.cs @@ -169,9 +169,10 @@ private async Task PostPointsAsync(string dbName, TimePrecision precision, line.Remove(line.Length - 1, 1); ByteArrayContent requestContent = new ByteArrayContent(Encoding.UTF8.GetBytes(line.ToString())); - var endPoint = new Dictionary() { - { "db", dbName }, - { "precision", precisionLiterals[(int)precision] } }; + var endPoint = new Dictionary() { { "db", dbName } }; + if (precision > 0) + endPoint.Add("precision", precisionLiterals[(int)precision]); + if (!String.IsNullOrWhiteSpace(retention)) endPoint.Add("rp", retention); HttpResponseMessage response = await PostAsync(endPoint, requestContent); @@ -195,11 +196,18 @@ private async Task PostPointsAsync(string dbName, TimePrecision precision, else parts = oneLinePattern.Matches(content.Substring(content.IndexOf("partial write:\\n") + 16)).ToList(); + if (parts.Count == 0) + throw new InfluxDBException("Partial Write", new Regex(@"\""error\"":\""(.*?)\""").Match(content).Groups[1].Value); + if (parts[1].Contains("\\n")) l = parts[1].Substring(0, parts[1].IndexOf("\\n")).Unescape(); else l = parts[1].Unescape(); } + catch (InfluxDBException e) + { + throw e; + } catch (Exception) { @@ -393,7 +401,7 @@ public async Task PostPointsAsync(string dbName, IEnumerable new { p.Precision, p.Retention?.Name })) + foreach (var group in Points.Where(p => p.Retention == null || p.UtcTimestamp > DateTime.UtcNow - p.Retention.Duration).GroupBy(p => new { p.Precision, p.Retention?.Name })) { var pointsGroup = group.AsEnumerable().Select((point, index) => new { Index = index, Point = point })//get the index of each point diff --git a/src/Extensions/EpochHelper.cs b/src/Extensions/EpochHelper.cs index 5d1c462..7c4d672 100644 --- a/src/Extensions/EpochHelper.cs +++ b/src/Extensions/EpochHelper.cs @@ -8,35 +8,35 @@ namespace AdysTech.InfluxDB.Client.Net { public static class EpochHelper { - private static readonly DateTime Origin = new DateTime (new DateTime (1970, 1, 1).Ticks, DateTimeKind.Utc); + private static readonly DateTime Origin = new DateTime(new DateTime(1970, 1, 1).Ticks, DateTimeKind.Utc); - public static long ToEpoch (this DateTime time, TimePrecision precision) + public static long ToEpoch(this DateTime time, TimePrecision precision) { TimeSpan t = time - Origin; switch (precision) { - case TimePrecision.Hours: return (long) t.TotalHours; - case TimePrecision.Minutes: return (long) t.TotalMinutes; - case TimePrecision.Seconds: return (long) t.TotalSeconds; - case TimePrecision.Milliseconds: return (long) t.TotalMilliseconds; - case TimePrecision.Microseconds: return (long) (t.TotalMilliseconds * 1000); - case TimePrecision.Nanoseconds: return (long) t.Ticks * 100; //1 tick = 100 nano sec + case TimePrecision.Hours: return (long)t.TotalHours; + case TimePrecision.Minutes: return (long)t.TotalMinutes; + case TimePrecision.Seconds: return (long)t.TotalSeconds; + case TimePrecision.Milliseconds: return (long)t.TotalMilliseconds; + case TimePrecision.Microseconds: return (long)(t.TotalMilliseconds * 1000); + case TimePrecision.Nanoseconds: + default: return (long)t.Ticks * 100; //1 tick = 100 nano sec } - return 0; } - public static DateTime FromEpoch (this string time, TimePrecision precision) + public static DateTime FromEpoch(this string time, TimePrecision precision) { - long duration = long.Parse (time); + long duration = long.Parse(time); DateTime t = Origin; switch (precision) { - case TimePrecision.Hours: return t.AddHours (duration); - case TimePrecision.Minutes: return t.AddMinutes (duration); - case TimePrecision.Seconds: return t.AddSeconds (duration); - case TimePrecision.Milliseconds: return t.AddMilliseconds (duration); - case TimePrecision.Microseconds: return t.AddTicks (duration * TimeSpan.TicksPerMillisecond * 1000); - case TimePrecision.Nanoseconds: return t.AddTicks (duration / 100); //1 tick = 100 nano sec + case TimePrecision.Hours: return t.AddHours(duration); + case TimePrecision.Minutes: return t.AddMinutes(duration); + case TimePrecision.Seconds: return t.AddSeconds(duration); + case TimePrecision.Milliseconds: return t.AddMilliseconds(duration); + case TimePrecision.Microseconds: return t.AddTicks(duration * TimeSpan.TicksPerMillisecond * 1000); + case TimePrecision.Nanoseconds: return t.AddTicks(duration / 100); //1 tick = 100 nano sec } return t; } diff --git a/tests/AdysTech.InfluxDB.Client.Net.Core.Test/AdysTech.InfluxDB.Client.Net.Core.Test.csproj b/tests/AdysTech.InfluxDB.Client.Net.Core.Test/AdysTech.InfluxDB.Client.Net.Core.Test.csproj index 2861a1f..1542ce1 100644 --- a/tests/AdysTech.InfluxDB.Client.Net.Core.Test/AdysTech.InfluxDB.Client.Net.Core.Test.csproj +++ b/tests/AdysTech.InfluxDB.Client.Net.Core.Test/AdysTech.InfluxDB.Client.Net.Core.Test.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/tests/InfluxDBClientTest.cs b/tests/InfluxDBClientTest.cs index 837acb3..5815ef9 100644 --- a/tests/InfluxDBClientTest.cs +++ b/tests/InfluxDBClientTest.cs @@ -667,5 +667,75 @@ public async Task TestQueryMultiSeriesAsync_Chunking_BySize() } } + [TestMethod, TestCategory("Post")] + public async Task TestPostPointsAsync_OlderthanRetention() + { + try + { + var client = new InfluxDBClient(influxUrl, dbUName, dbpwd); + var time = DateTime.Now; + var TestDate = time.ToShortDateString(); + var TestTime = time.ToShortTimeString(); + var points = new List(); + InfluxRetentionPolicy retention = new InfluxRetentionPolicy() { Duration = TimeSpan.FromHours(1) }; + + for (var i = 0; i < 10; i++) + { + await Task.Delay(1); + var point = new InfluxDatapoint(); + point.Retention = retention; + point.UtcTimestamp = DateTime.UtcNow.AddDays(-i); + point.MeasurementName = "RetentionTest"; + point.Precision = TimePrecision.Nanoseconds; + point.Tags.Add("TestDate", TestDate); + point.Tags.Add("TestTime", TestTime); + point.Fields.Add("Val", i); + points.Add(point); + } + + var r = await client.PostPointsAsync(dbName, points); + Assert.IsTrue(r, "PostPointsAsync retunred false"); + + Assert.IsTrue(points.Count(p => p.Saved) == 1, "PostPointsAsync saved points older than retention policy"); + + } + catch (Exception e) + { + Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}"); + return; + } + } + + [TestMethod, TestCategory("Post")] + public async Task TestPostPointsAsync_DefaultTimePrecision() + { + try + { + var client = new InfluxDBClient(influxUrl, dbUName, dbpwd); + var time = DateTime.Now; + var TestDate = time.ToShortDateString(); + var TestTime = time.ToShortTimeString(); + var points = new List(); + for (var i = 0; i < 10; i++) + { + await Task.Delay(1); + var point = new InfluxDatapoint(); + point.MeasurementName = "PrecisionTest"; + point.Tags.Add("TestDate", TestDate); + point.Tags.Add("TestTime", TestTime); + point.Fields.Add("Val", i); + points.Add(point); + } + + var r = await client.PostPointsAsync(dbName, points); + Assert.IsTrue(r, "PostPointsAsync retunred false"); + } + catch (Exception e) + { + Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}"); + return; + } + } + } } \ No newline at end of file