From 20eee97b23f1532fafb926ba5c0c0ea99d8e4c36 Mon Sep 17 00:00:00 2001 From: Sal Date: Mon, 29 May 2023 21:54:18 -0400 Subject: [PATCH 1/3] add universal exception handling setup basic analytics infrastructure remove newtonsoft package dependency! more spying on our users O.o add unit test for amplitude Better thrift metadata in metadata viewer update packages to fix #76 add logic to get user consent for analytics gathering respect consent setting nuke the api key --- src/Directory.Packages.props | 8 +- src/ParquetViewer.Engine/ParquetEngine.cs | 6 +- .../ParquetSchemaElement.cs | 2 +- .../Data/NULLABLE_GUID_TEST1.parquet | Bin 0 -> 6551 bytes .../ParquetViewer.Tests.csproj | 5 + src/ParquetViewer.Tests/SanityTests.cs | 91 +++++++- src/ParquetViewer.Tests/TestAmplitudeEvent.cs | 25 +++ src/ParquetViewer.Tests/Usings.cs | 3 +- src/ParquetViewer/AboutBox.Designer.cs | 2 +- src/ParquetViewer/AboutBox.cs | 2 +- src/ParquetViewer/Analytics/AllEvents.cs | 161 ++++++++++++++ src/ParquetViewer/Analytics/AmplitudeEvent.cs | 86 ++++++++ src/ParquetViewer/AppSettings.cs | 70 ++++++- .../Exceptions/InvalidQueryException.cs | 9 + src/ParquetViewer/Helpers/ExtensionMethods.cs | 5 +- .../Helpers/ParquetMetadataAnalyzers.cs | 197 ++++++++++++------ src/ParquetViewer/MainForm.Designer.cs | 32 ++- src/ParquetViewer/MainForm.EventHandlers.cs | 12 +- src/ParquetViewer/MainForm.Helpers.cs | 23 +- src/ParquetViewer/MainForm.ToolStripMenus.cs | 170 ++++++--------- src/ParquetViewer/MainForm.cs | 92 ++++---- src/ParquetViewer/MainForm.resx | 62 +++++- src/ParquetViewer/MetadataViewer.cs | 7 +- src/ParquetViewer/ParquetViewer.csproj | 1 - src/ParquetViewer/Program.cs | 68 +++++- src/ParquetViewer/Properties/AssemblyInfo.cs | 2 +- 26 files changed, 874 insertions(+), 267 deletions(-) create mode 100644 src/ParquetViewer.Tests/Data/NULLABLE_GUID_TEST1.parquet create mode 100644 src/ParquetViewer.Tests/TestAmplitudeEvent.cs create mode 100644 src/ParquetViewer/Analytics/AllEvents.cs create mode 100644 src/ParquetViewer/Analytics/AmplitudeEvent.cs create mode 100644 src/ParquetViewer/Exceptions/InvalidQueryException.cs diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 854cfd1..be4725e 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -3,14 +3,14 @@ true - - + - + + - + \ No newline at end of file diff --git a/src/ParquetViewer.Engine/ParquetEngine.cs b/src/ParquetViewer.Engine/ParquetEngine.cs index 8be2b09..fd942d3 100644 --- a/src/ParquetViewer.Engine/ParquetEngine.cs +++ b/src/ParquetViewer.Engine/ParquetEngine.cs @@ -12,6 +12,8 @@ public partial class ParquetEngine : IDisposable public long RecordCount => _recordCount ??= _parquetFiles.Sum(pf => pf.Metadata?.NumRows ?? 0); + public int NumberOfPartitions => _parquetFiles.Count; + private ParquetReader DefaultReader => _parquetFiles.FirstOrDefault() ?? throw new Exception("No parquet readers available"); public List Fields => DefaultReader.Schema.Fields.Select(f => f.Name).ToList() ?? new(); @@ -23,7 +25,7 @@ public partial class ParquetEngine : IDisposable public ParquetSchema Schema => DefaultReader.Schema ?? new(); private ParquetSchemaElement? _parquetSchemaTree; - private ParquetSchemaElement ParquetSchemaTree => _parquetSchemaTree ??= BuildParquetSchemaTree(); + public ParquetSchemaElement ParquetSchemaTree => _parquetSchemaTree ??= BuildParquetSchemaTree(); public string OpenFileOrFolderPath { get; } @@ -46,7 +48,7 @@ private ParquetSchemaElement BuildParquetSchemaTree() return thriftSchemaTree; } - private ParquetSchemaElement ReadSchemaTree(ref List.Enumerator schemaElements) + private static ParquetSchemaElement ReadSchemaTree(ref List.Enumerator schemaElements) { if (!schemaElements.MoveNext()) throw new Exception("Invalid parquet schema"); diff --git a/src/ParquetViewer.Engine/ParquetSchemaElement.cs b/src/ParquetViewer.Engine/ParquetSchemaElement.cs index 6df42ac..8b49c06 100644 --- a/src/ParquetViewer.Engine/ParquetSchemaElement.cs +++ b/src/ParquetViewer.Engine/ParquetSchemaElement.cs @@ -3,7 +3,7 @@ namespace ParquetViewer.Engine { - internal class ParquetSchemaElement + public class ParquetSchemaElement { public string Path => SchemaElement.Name; public SchemaElement SchemaElement { get; set; } diff --git a/src/ParquetViewer.Tests/Data/NULLABLE_GUID_TEST1.parquet b/src/ParquetViewer.Tests/Data/NULLABLE_GUID_TEST1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5943619b05f3182e3af938da7e6dc11d00207a16 GIT binary patch literal 6551 zcmb_hPiQ0886S-muVc%qv&l@yj!4vvJ@F=%I+A5sPSrz83U2CzWob)D1B$G9vWLhT zcOl`V*?>HCD}tL?IkQ^Nnt5z8#blC@6C+f zn;BaciOr0Gdn2nvjJhR9_6yUi}^e6Pp6 zZ`;O{L{R(^8Ici?6u+bf{EAi+@f$N-xbE)t*tXlT;UvL z*^m0n>oz-2vI|a&k>far9LIS{t(fDu0UXDck|=wk3E@%-TCu1tNu`?H-`_8GS--gL z8PYpIBubVbvIjQ&l1$=Vm?Ng5R+gZrNw(#>TOCH|Q4$WKjF=qIUw+niLt7vM1y9dw zFG!x|SKF*@m+V@>W)-_oDb-dB&01}_P+MKGnyt;s+H$#OjHHFQM$#J%4UrfBp8`y{3Q3qW%yL6|lA)j^Lt-WkKZAymrY9K+N-`v7(vVU!()1)l zK}m+hgke@YE!t59PdR1yIE30_hQUgVP}4g$w_aSGps zNo=E)mRFX`CDtsgHY==9S+h3_YZj{&+E$6R+SPJ(*=`y0qiCTFBk7HXhH}XpZIl2| zLRA}&bvC7>z<*YIUhs`GWvjeguvQAC)qKf%qP&8?MQhorSk_AZajX1*o#pLrB3JU@}WSBt6&9!z6NFGsd55Zh34o{0Y+-2WB&M zA7UKuzB*uz-ab_7*!H2buIKJB+jYI-p4TyE;v6_m@RC{xK;Axa5KW!Eb7np5xo%(k zCe{HigJ0@7eb(r)R^M^E4aZ(Mv+?@<`TGEb4Dy;{Ot1^A+lQocOE9N!#qb&}*L~4p zGJo3U7l4Rl9A2hc9&5snjfd}d+K265++|U(>Vj&d`_2xNBFTU^>FhR|w(YTAZ$a6} zg|Vo{eI9Ewk7A5Q%Cp<_+4z|<`OT)mlR@eMEJ;rLlI`#=MKQ|a@Mh8o5*A{L ze>N`exqDtKmMFOyL$77r49~?wz%jw8AQ4&lUVh%nn&f-R=k7ape_JXci88)un>kzC z{g^E4%j`X`Nh={Ho%E%#wkW{NI=x26xqv0qM~in9jnWEY;9r6;Vrs`^*pA3LBgy^l zE+fy~tw!%+ug`X(i-JJ2etTQnO@}YkpdAXVPqo}`4=cd;elM*u-qg@HnNM`wmYCNe=7xzdSq6N5C9xDRGL>$iE1a$AnxV+`g2eVD*lqZ zV#19OoJ)gxyhKX|H@>4LOY>u(gh@q@FF7xu963-@Ui@@BWDupYU7Zvj4mk$7n@>~v z?G!ZY$=NZePV-7eaveWI9b0mJB?H%?#tBTF=0R4h)uWGZ9sw=BgX}SVx&yqh zPjt$2OMhS8f$b4fCQ5aLggSF=Mqoz16de4$l>XNgbn3B@GDgD1aTBQd^V7mn_~qd~ z-Wb#09D}$%+8j8V79u*?+0%SW6Y-0$B+uY4s{Vlrh5E2jaF`6y-u-}SzG8@FCV#1U zY_zamF|e?P%#-HJvjQRVK_5W8JPQahOCn&2qILAS1QGl04?sRA5HVL0h@c4M)|`k# zx*iMwZp{He%$EcpC<6G~gE>tJj_24df`iobI&^}=^VOW_sC>KNn*M3@Y7R$2d|*s+ z5XcBx6yT+YqzuCYJ2-{t+jZ$7WH{`szAXV9cERDn6VmPDkZwFI(vRJPIEcS5Z#;~B zIm!$;nieW`f@oCHUq2#R9^Tn_{7bq8VbJThp%osT;x5PaemACH9W&p4q&~{zIf@Q) zJh|!AylINa2lvfEmv5SNnm!W~sQK=r!i?WYxyeCFznp@s9?qWOPsE&@LCudAMQ!C< zEXSg~^wDBa-z#T@yTR?2tN9!0%2{lr_~@AA&_+653vJhb`nawti{Cld^nSR9U%5WR z@T(%`^)P1SE5%Z&Sk5nQ?l~Phzp~nHwe8K?npLf^)snrs*=)1gTC>WmO0&dRt6Z(t RTFvrV<`eYgMs literal 0 HcmV?d00001 diff --git a/src/ParquetViewer.Tests/ParquetViewer.Tests.csproj b/src/ParquetViewer.Tests/ParquetViewer.Tests.csproj index 0b3f077..5537e52 100644 --- a/src/ParquetViewer.Tests/ParquetViewer.Tests.csproj +++ b/src/ParquetViewer.Tests/ParquetViewer.Tests.csproj @@ -20,6 +20,7 @@ + runtime; build; native; contentfiles; analyzers; buildtransitive @@ -33,6 +34,7 @@ + @@ -63,6 +65,9 @@ Always + + Always + Always diff --git a/src/ParquetViewer.Tests/SanityTests.cs b/src/ParquetViewer.Tests/SanityTests.cs index 33ddc2f..69b4fb6 100644 --- a/src/ParquetViewer.Tests/SanityTests.cs +++ b/src/ParquetViewer.Tests/SanityTests.cs @@ -1,4 +1,9 @@ +using ParquetViewer.Analytics; using ParquetViewer.Engine.Exceptions; +using RichardSzalay.MockHttp; +using System.Globalization; +using System.Net.Http.Json; +using System.Text.RegularExpressions; namespace ParquetViewer.Tests { @@ -7,7 +12,7 @@ public class SanityTests [Fact] public async Task DECIMALS_AND_BOOLS_TEST() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/DECIMALS_AND_BOOLS_TEST1.parquet", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/DECIMALS_AND_BOOLS_TEST1.parquet", default); Assert.Equal(30, parquetEngine.RecordCount); Assert.Equal(337, parquetEngine.Fields.Count); @@ -23,7 +28,7 @@ public async Task DECIMALS_AND_BOOLS_TEST() [Fact] public async Task DATETIME_TEST1_TEST() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/DATETIME_TEST1.parquet", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/DATETIME_TEST1.parquet", default); Assert.Equal(10, parquetEngine.RecordCount); Assert.Equal(3, parquetEngine.Fields.Count); @@ -37,7 +42,7 @@ public async Task DATETIME_TEST1_TEST() [Fact] public async Task DATETIME_TEST2_TEST() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/DATETIME_TEST2.parquet", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/DATETIME_TEST2.parquet", default); Assert.Equal(1, parquetEngine.RecordCount); Assert.Equal(11, parquetEngine.Fields.Count); @@ -59,7 +64,7 @@ public async Task DATETIME_TEST2_TEST() [Fact] public async Task RANDOM_TEST_FILE1_TEST() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/RANDOM_TEST_FILE1.parquet", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/RANDOM_TEST_FILE1.parquet", default); Assert.Equal(5, parquetEngine.RecordCount); Assert.Equal(42, parquetEngine.Fields.Count); @@ -77,7 +82,7 @@ public async Task RANDOM_TEST_FILE1_TEST() [Fact] public async Task SAME_COLUMN_NAME_DIFFERENT_CASING_TEST() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/SAME_COLUMN_NAME_DIFFERENT_CASING1.parquet", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/SAME_COLUMN_NAME_DIFFERENT_CASING1.parquet", default); Assert.Equal(14610, parquetEngine.RecordCount); Assert.Equal(12, parquetEngine.Fields.Count); @@ -96,7 +101,7 @@ public async Task MULTIPLE_SCHEMAS_DETECTED_TEST() [Fact] public async Task PARTITIONED_PARQUET_FILE_TEST() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/PARTITIONED_PARQUET_FILE_TEST1", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/PARTITIONED_PARQUET_FILE_TEST1", default); Assert.Equal(2000, parquetEngine.RecordCount); Assert.Equal(9, parquetEngine.Fields.Count); @@ -121,7 +126,7 @@ public async Task PARTITIONED_PARQUET_FILE_TEST() [Fact] public async Task COLUMN_ENDING_IN_PERIOD_TEST1() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/COLUMN_ENDING_IN_PERIOD_TEST1.parquet", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/COLUMN_ENDING_IN_PERIOD_TEST1.parquet", default); Assert.Equal(1, parquetEngine.RecordCount); Assert.Equal(11, parquetEngine.Fields.Count); @@ -135,7 +140,7 @@ public async Task COLUMN_ENDING_IN_PERIOD_TEST1() [Fact] public async Task LIST_TYPE_TEST() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/LIST_TYPE_TEST1.parquet", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/LIST_TYPE_TEST1.parquet", default); Assert.Equal(3, parquetEngine.RecordCount); Assert.Equal(2, parquetEngine.Fields.Count); @@ -157,7 +162,7 @@ public async Task LIST_TYPE_TEST() [Fact] public async Task MAP_TYPE_TEST1() { - var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/MAP_TYPE_TEST1.parquet", default); + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/MAP_TYPE_TEST1.parquet", default); Assert.Equal(2, parquetEngine.RecordCount); Assert.Equal(2, parquetEngine.Fields.Count); @@ -172,5 +177,73 @@ public async Task MAP_TYPE_TEST1() Assert.Equal("value2", ((MapValue)dataTable.Rows[1][0]).Key); Assert.Equal("else", ((MapValue)dataTable.Rows[1][0]).Value); } + + [Fact] + public async Task AMPLITUDE_EVENT_TEST() + { + const string dummyApiKeyBase64 = "ZHVtbXk="; + var testEvent = new TestAmplitudeEvent(dummyApiKeyBase64) + { + IgnoredProperty = "xxx", + RegularProperty = "yyy" + }; + + string expectedRequestJson = @$" +{{ + ""api_key"": ""dummy"", + ""events"": [{{ + ""device_id"": ""{AppSettings.AnalyticsDeviceId}"", + ""event_type"": ""{TestAmplitudeEvent.EVENT_TYPE}"", + ""user_properties"": {{ + ""rememberLastRowCount"": {AppSettings.RememberLastRowCount.ToString().ToLower()}, + ""lastRowCount"": {AppSettings.LastRowCount}, + ""alwaysSelectAllFields"": {AppSettings.AlwaysSelectAllFields.ToString().ToLower()}, + ""autoSizeColumnsMode"": ""{AppSettings.AutoSizeColumnsMode}"", + ""dateTimeDisplayFormat"": ""{AppSettings.DateTimeDisplayFormat}"", + ""systemMemory"": {(int)(GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 1048576.0 /*magic number*/)}, + ""processorCount"": {Environment.ProcessorCount} + }}, + ""event_properties"": {{ + ""regularProperty"": ""yyy"" + }}, + ""session_id"": {testEvent.SessionId}, + ""language"": ""{CultureInfo.CurrentUICulture.Name}"", + ""os_name"": ""{Environment.OSVersion.Platform}"", + ""os_version"": ""{Environment.OSVersion.VersionString}"", + ""app_version"": ""{AboutBox.AssemblyVersion}"" + }}] +}}"; + + //mock the http request + var mockHttpHandler = new MockHttpMessageHandler(); + _ = mockHttpHandler.Expect(HttpMethod.Post, "*").Respond(async (request) => + { + //Verify the request we're sending is what we expect it to be + string requestJsonBody = await (request.Content?.ReadAsStringAsync() ?? Task.FromResult(string.Empty)); + if (Regex.Replace(requestJsonBody, "\\s", string.Empty) + .Equals(Regex.Replace(expectedRequestJson, "\\s", string.Empty))) + return new HttpResponseMessage(System.Net.HttpStatusCode.OK); + else + return new HttpResponseMessage(System.Net.HttpStatusCode.BadRequest); + }); + testEvent.SwapHttpClientHandler(mockHttpHandler); + + bool wasSuccess = await testEvent.Record(); + Assert.True(wasSuccess, "The event json we would have sent to Amplitude didn't match the expected value"); + } + + [Fact] + public async Task NULLABLE_GUID_TEST1() + { + using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/NULLABLE_GUID_TEST1.parquet", default); + + Assert.Equal(1, parquetEngine.RecordCount); + Assert.Equal(33, parquetEngine.Fields.Count); + + var dataTable = await parquetEngine.ReadRowsAsync(parquetEngine.Fields, 0, int.MaxValue, default); + Assert.Equal(false, dataTable.Rows[0][22]); + Assert.Equal(new Guid("0cf9cbfd-d320-45d7-b29f-9c2de1baa979"), dataTable.Rows[0][1]); + Assert.Equal(new DateTime(2019, 1, 1), dataTable.Rows[0][4]); + } } } diff --git a/src/ParquetViewer.Tests/TestAmplitudeEvent.cs b/src/ParquetViewer.Tests/TestAmplitudeEvent.cs new file mode 100644 index 0000000..1be63e4 --- /dev/null +++ b/src/ParquetViewer.Tests/TestAmplitudeEvent.cs @@ -0,0 +1,25 @@ +using ParquetViewer.Analytics; +using System.Text.Json.Serialization; + +namespace ParquetViewer.Tests +{ + public class TestAmplitudeEvent : AmplitudeEvent + { + public const string EVENT_TYPE = "unit.test.event"; + + [JsonIgnore] + public string? IgnoredProperty { get; set; } + + public string? RegularProperty { get; set; } + + public TestAmplitudeEvent(string dummyApiKey) : base(EVENT_TYPE) + { + base.AMPLITUDE_API_KEY = dummyApiKey; + } + + public void SwapHttpClientHandler(HttpMessageHandler mockHandler) + { + HttpMessageHandler = mockHandler; + } + } +} diff --git a/src/ParquetViewer.Tests/Usings.cs b/src/ParquetViewer.Tests/Usings.cs index dc754b5..df9c817 100644 --- a/src/ParquetViewer.Tests/Usings.cs +++ b/src/ParquetViewer.Tests/Usings.cs @@ -1,3 +1,2 @@ -global using Xunit; global using ParquetViewer.Engine; -global using System.Diagnostics; \ No newline at end of file +global using Xunit; diff --git a/src/ParquetViewer/AboutBox.Designer.cs b/src/ParquetViewer/AboutBox.Designer.cs index 5d18436..0159558 100644 --- a/src/ParquetViewer/AboutBox.Designer.cs +++ b/src/ParquetViewer/AboutBox.Designer.cs @@ -1,6 +1,6 @@ namespace ParquetViewer { - partial class AboutBox + public partial class AboutBox { /// /// Required designer variable. diff --git a/src/ParquetViewer/AboutBox.cs b/src/ParquetViewer/AboutBox.cs index dcadfad..67d52d0 100644 --- a/src/ParquetViewer/AboutBox.cs +++ b/src/ParquetViewer/AboutBox.cs @@ -4,7 +4,7 @@ namespace ParquetViewer { - partial class AboutBox : Form + public partial class AboutBox : Form { public AboutBox() { diff --git a/src/ParquetViewer/Analytics/AllEvents.cs b/src/ParquetViewer/Analytics/AllEvents.cs new file mode 100644 index 0000000..5fc1ec4 --- /dev/null +++ b/src/ParquetViewer/Analytics/AllEvents.cs @@ -0,0 +1,161 @@ +using System.Text.Json.Serialization; + +namespace ParquetViewer.Analytics +{ + public class ProgramOpenEvent : AmplitudeEvent + { + private const string EVENT_TYPE = "program.open"; + + public bool IsOpeningFile { get; set; } + + public ProgramOpenEvent() : base(EVENT_TYPE) + { + + } + + public static void FireAndForget(bool isOpeningFile) + { + var _ = new ProgramOpenEvent { IsOpeningFile = isOpeningFile }.Record(); + } + } + + public class FileOpenEvent : AmplitudeEvent + { + private const string EVENT_TYPE = "file.open"; + + public bool IsFolder { get; set; } + public int NumPartitions { get; set; } + public long NumRows { get; set; } + public int NumRowGroups { get; set; } + public int NumFields { get; set; } + public string[] FieldTypes { get; set; } + public long RecordOffset { get; set; } + public long RecordCount { get; set; } + public int NumLoadedFields { get; set; } + public long LoadTimeMS { get; set; } + + public FileOpenEvent() : base(EVENT_TYPE) + { + + } + + public static void FireAndForget(bool isFolder, int numPartitions, long numRows, int numRowGroups, int numFields, + string[] fieldTypes, long recordOffset, long recordCount, int numLoadedFields, long loadTimeMilliseconds) + { + var _ = new FileOpenEvent + { + IsFolder = isFolder, + NumPartitions = numPartitions, + NumRows = numRows, + NumRowGroups = numRowGroups, + NumFields = numFields, + FieldTypes = fieldTypes, + RecordOffset = recordOffset, + RecordCount = recordCount, + NumLoadedFields = numLoadedFields, + LoadTimeMS = loadTimeMilliseconds + }.Record(); + } + } + + public class FileExportEvent : AmplitudeEvent + { + private const string EVENT_TYPE = "file.saveas"; + + [JsonIgnore] + public Helpers.FileType FileType { get; set; } + public string FileTypeName => FileType.ToString(); + public long FileSize { get; set; } + public long RowCount { get; set; } + public int ColumnCount { get; set; } + public long ExportTimeMS { get; set; } + + public FileExportEvent() : base(EVENT_TYPE) + { + + } + + public static void FireAndForget(Helpers.FileType fileType, long fileSize, int rowCount, int columnCount, long exportTimeInMilliseconds) + { + var _ = new FileExportEvent + { + FileType = fileType, + FileSize = fileSize, + RowCount = rowCount, + ColumnCount = columnCount, + ExportTimeMS = exportTimeInMilliseconds + }.Record(); + } + } + + public class MenuBarClickEvent : AmplitudeEvent + { + private const string EVENT_TYPE = "menubar.click"; + + [JsonIgnore] + public ActionId Action { get; set; } + + public string ActionName => Action.ToString(); + + public MenuBarClickEvent() : base(EVENT_TYPE) + { + + } + + public static void FireAndForget(ActionId action) + { + var _ = new MenuBarClickEvent { Action = action }.Record(); + } + + public enum ActionId + { + None = 0, + FileNew, + FileOpen, + FolderOpen, + Exit, + ChangeFields, + SQLCreateTable, + MetadataViewer, + AboutBox, + UserGuide, + DragDrop + } + } + + public class ExecuteQueryEvent : AmplitudeEvent + { + private const string EVENT_TYPE = "sql.execute"; + + public bool IsValid { get; set; } + public int RecordCount { get; set; } + public int ColumnCount { get; set; } + public long RunTimeMS { get; set; } + + public ExecuteQueryEvent() : base(EVENT_TYPE) + { + + } + } + + public class ExceptionEvent : AmplitudeEvent + { + private const string EVENT_TYPE = "exception.thrown"; + + [JsonIgnore] + public System.Exception Exception { get; set; } + + public string Message => Exception?.Message; + public string StackTrace => Exception?.StackTrace?.ToString(); + + public ExceptionEvent() : base(EVENT_TYPE) + { + + } + + public static void FireAndForget(System.Exception ex) + { + var _ = new ExceptionEvent { Exception = ex }.Record(); + } + } +} diff --git a/src/ParquetViewer/Analytics/AmplitudeEvent.cs b/src/ParquetViewer/Analytics/AmplitudeEvent.cs new file mode 100644 index 0000000..985f862 --- /dev/null +++ b/src/ParquetViewer/Analytics/AmplitudeEvent.cs @@ -0,0 +1,86 @@ +using ParquetViewer.Helpers; +using System; +using System.Globalization; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace ParquetViewer.Analytics +{ + public abstract class AmplitudeEvent + { + //The api key is meant to be public: https://www.docs.developers.amplitude.com/guides/amplitude-keys-guide/#api-key + protected string AMPLITUDE_API_KEY = ""; //This will only be populated for official releases + + private static readonly long _sessionId = DateTime.UtcNow.ToMillisecondsSinceEpoch(); + private static readonly int _systemRAM = (int)(GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 1048576.0 /*magic number*/); + + protected HttpMessageHandler HttpMessageHandler { get; set; } = new HttpClientHandler(); + + [JsonIgnore] + public string DeviceId => AppSettings.AnalyticsDeviceId.ToString(); + + [JsonIgnore] + public string EventType { get; } + + [JsonIgnore] + public long SessionId => _sessionId; + + [JsonIgnore] + public object UserProperties => new + { + AppSettings.RememberLastRowCount, + AppSettings.LastRowCount, + AppSettings.AlwaysSelectAllFields, + AutoSizeColumnsMode = AppSettings.AutoSizeColumnsMode.ToString(), + DateTimeDisplayFormat = AppSettings.DateTimeDisplayFormat.ToString(), + SystemMemory = _systemRAM, + Environment.ProcessorCount + }; + + protected AmplitudeEvent(string eventType) + { + EventType = eventType; + } + + public async Task Record() + { + try + { + if (AMPLITUDE_API_KEY.Length == 0 || !AppSettings.AnalyticsDataGatheringConsent) + return false; + + var request = new + { + api_key = string.Join(string.Empty, Base64Decode(AMPLITUDE_API_KEY)), + events = new[] { + new { + device_id = DeviceId, + event_type = EventType, + user_properties = UserProperties, + event_properties = Convert.ChangeType(this, GetType()), //If we don't cast it to the correct child type, its properties don't get picked up + session_id = _sessionId, + language = CultureInfo.CurrentUICulture.Name, + os_name = Environment.OSVersion.Platform.ToString(), + os_version = Environment.OSVersion.VersionString, + app_version = AboutBox.AssemblyVersion + } + } + }; + + var result = await new HttpClient(this.HttpMessageHandler).PostAsync("https://api2.amplitude.com/2/httpapi", JsonContent.Create(request)); + return result.IsSuccessStatusCode; + } + catch { /* Analytics is best effort. If it fails, it fails */ } + + return false; + } + + private static string Base64Decode(string base64EncodedData) + { + var base64EncodedBytes = Convert.FromBase64String(base64EncodedData); + return System.Text.Encoding.UTF8.GetString(base64EncodedBytes); + } + } +} \ No newline at end of file diff --git a/src/ParquetViewer/AppSettings.cs b/src/ParquetViewer/AppSettings.cs index e73614c..f5759ff 100644 --- a/src/ParquetViewer/AppSettings.cs +++ b/src/ParquetViewer/AppSettings.cs @@ -2,6 +2,8 @@ using ParquetViewer.Helpers; using System; +#nullable enable + namespace ParquetViewer { public static class AppSettings @@ -14,7 +16,9 @@ public static class AppSettings private const string ParquetReadingEngineKey = "ParquetReadingEngine"; private const string AutoSizeColumnsModeKey = "AutoSizeColumnsMode"; private const string DateTimeDisplayFormatKey = "DateTimeDisplayFormat"; - private const string WarningBypassedOnVersionKey = "WarningBypassedOnVersion"; + private const string ConsentLastAskedOnVersionKey = "ConsentLastAskedOnVersion"; + private const string AnalyticsDeviceIdKey = "AnalyticsDeviceId"; + private const string AnalyticsDataGatheringConsentKey = "AnalyticsDataGatheringConsent"; //TODO: Cleanup this setting after sufficient time has passed. [Obsolete($"We have more date formats now so use {nameof(DateTimeDisplayFormat)} instead.")] @@ -256,7 +260,7 @@ public static AutoSizeColumnsMode AutoSizeColumnsMode } } - public static string? WarningBypassedOnVersion + public static string? ConsentLastAskedOnVersion { get { @@ -264,7 +268,7 @@ public static string? WarningBypassedOnVersion { using (RegistryKey registryKey = Registry.CurrentUser.CreateSubKey(RegistrySubKey)) { - return registryKey.GetValue(WarningBypassedOnVersionKey)?.ToString(); + return registryKey.GetValue(ConsentLastAskedOnVersionKey)?.ToString(); } } catch @@ -278,7 +282,65 @@ public static string? WarningBypassedOnVersion { using (RegistryKey registryKey = Registry.CurrentUser.CreateSubKey(RegistrySubKey)) { - registryKey.SetValue(WarningBypassedOnVersionKey, value ?? string.Empty); + registryKey.SetValue(ConsentLastAskedOnVersionKey, value ?? string.Empty); + } + } + catch { } + } + } + + public static Guid AnalyticsDeviceId + { + get + { + try + { + using (RegistryKey registryKey = Registry.CurrentUser.CreateSubKey(RegistrySubKey)) + { + if (Guid.TryParse(registryKey.GetValue(AnalyticsDeviceIdKey)?.ToString(), out var value)) + { + return value; + } + else + { + //This user doesn't have an analytics device id yet, so create one + Guid newDeviceId = Guid.NewGuid(); + registryKey.SetValue(AnalyticsDeviceIdKey, newDeviceId); + return newDeviceId; + } + } + } + catch + { + return Guid.Empty; + } + } + } + + public static bool AnalyticsDataGatheringConsent + { + get + { + try + { + using (RegistryKey registryKey = Registry.CurrentUser.CreateSubKey(RegistrySubKey)) + { + bool.TryParse(registryKey.GetValue(AnalyticsDataGatheringConsentKey)?.ToString(), out var value); + return value; + } + } + catch + { + return false; + } + } + set + { + try + { + using (RegistryKey registryKey = Registry.CurrentUser.CreateSubKey(RegistrySubKey)) + { + registryKey.SetValue(AnalyticsDataGatheringConsentKey, value.ToString()); } } catch { } diff --git a/src/ParquetViewer/Exceptions/InvalidQueryException.cs b/src/ParquetViewer/Exceptions/InvalidQueryException.cs new file mode 100644 index 0000000..5e9f061 --- /dev/null +++ b/src/ParquetViewer/Exceptions/InvalidQueryException.cs @@ -0,0 +1,9 @@ +using System; + +namespace ParquetViewer.Exceptions +{ + public class InvalidQueryException : Exception + { + public InvalidQueryException(Exception ex = null) : base("The query doesn't seem to be valid. Please try again.", ex) { } + } +} diff --git a/src/ParquetViewer/Helpers/ExtensionMethods.cs b/src/ParquetViewer/Helpers/ExtensionMethods.cs index e6f9200..6eb273e 100644 --- a/src/ParquetViewer/Helpers/ExtensionMethods.cs +++ b/src/ParquetViewer/Helpers/ExtensionMethods.cs @@ -1,4 +1,5 @@ -using ParquetViewer.Engine; +using Parquet.Rows; +using ParquetViewer.Engine; using System; using System.Collections.Generic; using System.Data; @@ -73,5 +74,7 @@ public static IList GetColumnNames(this DataTable datatable) FileType.XLS => ".xls", _ => throw new ArgumentOutOfRangeException(nameof(fileType)) }; + + public static long ToMillisecondsSinceEpoch(this DateTime dateTime) => new DateTimeOffset(dateTime).ToUnixTimeMilliseconds(); } } diff --git a/src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs b/src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs index 5fac412..843cbbf 100644 --- a/src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs +++ b/src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs @@ -1,9 +1,10 @@ using Apache.Arrow.Ipc; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Parquet.Meta; using System; using System.Linq; +using System.Text.Json; + +#nullable enable namespace ParquetViewer.Helpers { @@ -14,94 +15,172 @@ public static string ApacheArrowToJSON(string base64) try { byte[] bytes = Convert.FromBase64String(base64); - using (ArrowStreamReader reader = new ArrowStreamReader(bytes)) + using (ArrowStreamReader reader = new(bytes)) { reader.ReadNextRecordBatch(); - return JsonConvert.SerializeObject(reader.Schema, Formatting.Indented); + return JsonSerializer.Serialize(reader.Schema, new JsonSerializerOptions { WriteIndented = true }); } } catch (Exception ex) { - return $"Something went wrong while processing the schema:{Environment.NewLine}{Environment.NewLine}{ex.ToString()}"; + return $"Something went wrong while processing the schema:{Environment.NewLine}{Environment.NewLine}{ex}"; } } - public static string ThriftMetadataToJSON(FileMetaData thriftMetadata, long recordCount, int fieldCount) + public static string ThriftMetadataToJSON(Engine.ParquetEngine parquetEngine, long recordCount, int fieldCount) { try { - var jsonObject = new JObject - { - [nameof(thriftMetadata.Version)] = thriftMetadata.Version, - [nameof(thriftMetadata.NumRows)] = recordCount, - ["NumRowGroups"] = thriftMetadata.RowGroups?.Count ?? -1, //What about partitioned files? - ["NumFields"] = fieldCount, - [nameof(thriftMetadata.CreatedBy)] = thriftMetadata.CreatedBy - }; - - var schemas = new JArray(); - foreach (var schema in thriftMetadata.Schema) + object ProcessSchemaTree(Engine.ParquetSchemaElement parquetSchemaElement) { - if ("schema".Equals(schema.Name) && schemas.Count == 0) - continue; - - var schemaObject = new JObject + return new { - [nameof(schema.FieldId)] = schema.FieldId, - [nameof(schema.Name)] = schema.Name, - [nameof(schema.Type)] = schema.Type.ToString(), - [nameof(schema.TypeLength)] = schema.TypeLength, - [nameof(schema.LogicalType)] = schema.LogicalType?.ToString(), - [nameof(schema.Scale)] = schema.Scale, - [nameof(schema.Precision)] = schema.Precision, - [nameof(schema.RepetitionType)] = schema.RepetitionType.ToString(), - [nameof(schema.ConvertedType)] = schema.ConvertedType.ToString() + parquetSchemaElement.Path, + Type = parquetSchemaElement.SchemaElement.Type.ToString(), + parquetSchemaElement.SchemaElement.TypeLength, + LogicalType = LogicalTypeToJSONObject(parquetSchemaElement.SchemaElement.LogicalType), + RepetitionType = parquetSchemaElement.SchemaElement.RepetitionType.ToString(), + ConvertedType = parquetSchemaElement.SchemaElement.ConvertedType.ToString(), + Children = parquetSchemaElement.Children.Select(pse => ProcessSchemaTree(pse)).ToArray() }; - - schemas.Add(schemaObject); } - jsonObject[nameof(thriftMetadata.Schema)] = schemas; - var rowGroups = new JArray(); - foreach (var rowGroup in thriftMetadata.RowGroups ?? Enumerable.Empty()) + var jsonObject = new { - var rowGroupObject = new JObject(); - rowGroupObject[nameof(rowGroup.Ordinal)] = rowGroup.Ordinal; - rowGroupObject[nameof(rowGroup.NumRows)] = rowGroup.NumRows; - - var sortingColumns = new JArray(); - foreach (var sortingColumn in rowGroup.SortingColumns ?? Enumerable.Empty()) + parquetEngine.ThriftMetadata.Version, + NumRows = recordCount, + NumRowGroups = parquetEngine.ThriftMetadata.RowGroups?.Count ?? -1, //What about partitioned files? + NumFields = fieldCount, + parquetEngine.ThriftMetadata.CreatedBy, + Schema = ProcessSchemaTree(parquetEngine.ParquetSchemaTree), + RowGroups = (parquetEngine.ThriftMetadata.RowGroups ?? Enumerable.Empty()).Select(rowGroup => new { - var sortingColumnObject = new JObject(); - sortingColumnObject[nameof(sortingColumn.ColumnIdx)] = sortingColumn.ColumnIdx; - sortingColumnObject[nameof(sortingColumn.Descending)] = sortingColumn.Descending; - sortingColumnObject[nameof(sortingColumn.NullsFirst)] = sortingColumn.NullsFirst; - - sortingColumns.Add(sortingColumnObject); - } - - rowGroupObject[nameof(rowGroup.SortingColumns)] = sortingColumns; - rowGroupObject[nameof(rowGroup.FileOffset)] = rowGroup.FileOffset; - rowGroupObject[nameof(rowGroup.TotalByteSize)] = rowGroup.TotalByteSize; - rowGroupObject[nameof(rowGroup.TotalCompressedSize)] = rowGroup.TotalCompressedSize; - - rowGroups.Add(rowGroupObject); - } - jsonObject[nameof(thriftMetadata.RowGroups)] = rowGroups; + rowGroup.Ordinal, + rowGroup.NumRows, + SortingColumns = (rowGroup.SortingColumns ?? Enumerable.Empty()).Select(sortingColumn => new + { + sortingColumn.ColumnIdx, + sortingColumn.Descending, + sortingColumn.NullsFirst + }).ToArray(), + rowGroup.FileOffset, + rowGroup.TotalByteSize, + rowGroup.TotalCompressedSize + }).ToArray() + }; - return jsonObject.ToString(Formatting.Indented); + return JsonSerializer.Serialize(jsonObject, new JsonSerializerOptions { WriteIndented = true }); } catch (Exception ex) { return $"Something went wrong while processing the schema:{Environment.NewLine}{Environment.NewLine}{ex}"; } + + static object? LogicalTypeToJSONObject(LogicalType? logicalType) + { + if (logicalType is null) + { + return null; + } + else if (logicalType.STRING is not null) + { + return new { Name = nameof(logicalType.STRING) }; + } + else if (logicalType.MAP is not null) + { + return new { Name = nameof(logicalType.MAP) }; + } + else if (logicalType.LIST is not null) + { + return new { Name = nameof(logicalType.LIST) }; + } + else if (logicalType.ENUM is not null) + { + return new { Name = nameof(logicalType.ENUM) }; + } + else if (logicalType.DECIMAL is not null) + { + return new + { + Name = nameof(logicalType.DECIMAL), + logicalType.DECIMAL.Scale, + logicalType.DECIMAL.Precision + }; + } + else if (logicalType.DATE is not null) + { + return new { Name = nameof(logicalType.DATE) }; + } + else if (logicalType.TIME is not null) + { + return new + { + Name = nameof(logicalType.TIME), + logicalType.TIME.IsAdjustedToUTC, + Unit = TimeUnitToString(logicalType.TIME.Unit) + }; + } + else if (logicalType.TIMESTAMP is not null) + { + return new + { + Name = nameof(logicalType.TIMESTAMP), + logicalType.TIMESTAMP.IsAdjustedToUTC, + Unit = TimeUnitToString(logicalType.TIMESTAMP.Unit) + }; + } + else if (logicalType.INTEGER is not null) + { + return new + { + Name = nameof(logicalType.INTEGER), + logicalType.INTEGER.BitWidth, + logicalType.INTEGER.IsSigned + }; + } + else if (logicalType.JSON is not null) + { + return new { Name = nameof(logicalType.JSON) }; + } + else if (logicalType.BSON is not null) + { + return new { Name = nameof(logicalType.BSON) }; + } + else if (logicalType.UUID is not null) + { + return new { Name = nameof(logicalType.UUID) }; + } + else + { + return new { Name = nameof(logicalType.UNKNOWN) }; + } + } + + static string TimeUnitToString(TimeUnit? timeUnit) + { + var timeUnitString = string.Empty; + if (timeUnit?.MILLIS is not null) + { + timeUnitString = nameof(timeUnit.MILLIS); + } + else if (timeUnit?.MICROS is not null) + { + timeUnitString = nameof(timeUnit.MICROS); + } + else if (timeUnit?.NANOS is not null) + { + timeUnitString = nameof(timeUnit.NANOS); + } + return timeUnitString; + } } public static string TryFormatJSON(string possibleJSON) { try { - return JToken.Parse(possibleJSON).ToString(Formatting.Indented); + var jsonElement = JsonSerializer.Deserialize(possibleJSON); + return JsonSerializer.Serialize(jsonElement, new JsonSerializerOptions { WriteIndented = true }); } catch (Exception) { diff --git a/src/ParquetViewer/MainForm.Designer.cs b/src/ParquetViewer/MainForm.Designer.cs index ac1344f..47f8874 100644 --- a/src/ParquetViewer/MainForm.Designer.cs +++ b/src/ParquetViewer/MainForm.Designer.cs @@ -73,6 +73,7 @@ private void InitializeComponent() metadataViewerToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); helpToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); userGuideToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + shareAnonymousUsageDataToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); aboutToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); showingRecordCountStatusBarLabel = new System.Windows.Forms.ToolStripStatusLabel(); actualShownRecordCountLabel = new System.Windows.Forms.ToolStripStatusLabel(); @@ -237,11 +238,28 @@ private void InitializeComponent() // // mainGridView // + mainGridView.AllowUserToAddRows = false; + mainGridView.AllowUserToDeleteRows = false; + mainGridView.AllowUserToOrderColumns = true; mainGridView.Anchor = System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right; + mainGridView.ClipboardCopyMode = System.Windows.Forms.DataGridViewClipboardCopyMode.EnableWithoutHeaderText; + mainGridView.ColumnHeadersBorderStyle = System.Windows.Forms.DataGridViewHeaderBorderStyle.Single; + dataGridViewCellStyle1.Alignment = System.Windows.Forms.DataGridViewContentAlignment.MiddleLeft; + dataGridViewCellStyle1.BackColor = System.Drawing.SystemColors.ControlLight; + dataGridViewCellStyle1.Font = new System.Drawing.Font("Segoe UI Semibold", 9F, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point); + dataGridViewCellStyle1.ForeColor = System.Drawing.SystemColors.WindowText; + dataGridViewCellStyle1.SelectionBackColor = System.Drawing.SystemColors.Highlight; + dataGridViewCellStyle1.SelectionForeColor = System.Drawing.SystemColors.HighlightText; + dataGridViewCellStyle1.WrapMode = System.Windows.Forms.DataGridViewTriState.True; + mainGridView.ColumnHeadersDefaultCellStyle = dataGridViewCellStyle1; + mainGridView.ColumnHeadersHeightSizeMode = System.Windows.Forms.DataGridViewColumnHeadersHeightSizeMode.DisableResizing; mainTableLayoutPanel.SetColumnSpan(mainGridView, 10); + mainGridView.EnableHeadersVisualStyles = false; mainGridView.Location = new System.Drawing.Point(4, 38); mainGridView.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); mainGridView.Name = "mainGridView"; + mainGridView.ReadOnly = true; + mainGridView.RowHeadersWidth = 24; mainTableLayoutPanel.SetRowSpan(mainGridView, 2); mainGridView.Size = new System.Drawing.Size(936, 356); mainGridView.TabIndex = 6; @@ -476,7 +494,7 @@ private void InitializeComponent() // // helpToolStripMenuItem // - helpToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { userGuideToolStripMenuItem, aboutToolStripMenuItem }); + helpToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { userGuideToolStripMenuItem, shareAnonymousUsageDataToolStripMenuItem, aboutToolStripMenuItem }); helpToolStripMenuItem.Name = "helpToolStripMenuItem"; helpToolStripMenuItem.Size = new System.Drawing.Size(44, 20); helpToolStripMenuItem.Text = "&Help"; @@ -484,14 +502,21 @@ private void InitializeComponent() // userGuideToolStripMenuItem // userGuideToolStripMenuItem.Name = "userGuideToolStripMenuItem"; - userGuideToolStripMenuItem.Size = new System.Drawing.Size(131, 22); + userGuideToolStripMenuItem.Size = new System.Drawing.Size(180, 22); userGuideToolStripMenuItem.Text = "User Guide"; userGuideToolStripMenuItem.Click += userGuideToolStripMenuItem_Click; // + // shareAnonymousUsageDataToolStripMenuItem + // + shareAnonymousUsageDataToolStripMenuItem.Name = "shareAnonymousUsageDataToolStripMenuItem"; + shareAnonymousUsageDataToolStripMenuItem.Size = new System.Drawing.Size(180, 22); + shareAnonymousUsageDataToolStripMenuItem.Text = "Share Usage Data"; + shareAnonymousUsageDataToolStripMenuItem.Click += shareAnonymousUsageDataToolStripMenuItem_Click; + // // aboutToolStripMenuItem // aboutToolStripMenuItem.Name = "aboutToolStripMenuItem"; - aboutToolStripMenuItem.Size = new System.Drawing.Size(131, 22); + aboutToolStripMenuItem.Size = new System.Drawing.Size(180, 22); aboutToolStripMenuItem.Text = "&About..."; aboutToolStripMenuItem.Click += aboutToolStripMenuItem_Click; // @@ -653,6 +678,7 @@ private void InitializeComponent() private System.Windows.Forms.ToolStripMenuItem rememberRecordCountToolStripMenuItem; private System.Windows.Forms.FolderBrowserDialog openFolderDialog; private System.Windows.Forms.ToolStripMenuItem openFolderToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem shareAnonymousUsageDataToolStripMenuItem; } } diff --git a/src/ParquetViewer/MainForm.EventHandlers.cs b/src/ParquetViewer/MainForm.EventHandlers.cs index 43a6cf6..8687c48 100644 --- a/src/ParquetViewer/MainForm.EventHandlers.cs +++ b/src/ParquetViewer/MainForm.EventHandlers.cs @@ -1,11 +1,14 @@ -using System; +using ParquetViewer.Analytics; +using System; +using System.Text.RegularExpressions; using System.Windows.Forms; namespace ParquetViewer { public partial class MainForm { - private const string QueryUselessPartRegex = "^WHERE "; + [GeneratedRegex("^WHERE ")] + private static partial Regex QueryUselessPartRegex(); private void offsetTextBox_KeyPress(object sender, KeyPressEventArgs e) { @@ -57,13 +60,14 @@ private async void MainForm_DragDrop(object sender, DragEventArgs e) if (files != null && files.Length > 0) { this.Cursor = Cursors.WaitCursor; + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.DragDrop); await this.OpenNewFileOrFolder(files[0]); } } - catch (Exception ex) + catch { this.OpenFileOrFolderPath = null; - ShowError(ex); + throw; } } diff --git a/src/ParquetViewer/MainForm.Helpers.cs b/src/ParquetViewer/MainForm.Helpers.cs index 7474b2b..ca03e03 100644 --- a/src/ParquetViewer/MainForm.Helpers.cs +++ b/src/ParquetViewer/MainForm.Helpers.cs @@ -1,7 +1,9 @@ -using ParquetViewer.Engine.Exceptions; +using ParquetViewer.Analytics; +using ParquetViewer.Engine.Exceptions; using ParquetViewer.Helpers; using System; using System.Data; +using System.Diagnostics; using System.IO; using System.Linq; using System.Text; @@ -31,11 +33,6 @@ private LoadingIcon ShowLoadingIcon(string message, long loadingBarMax = 0) return loadingIcon; } - private static void ShowError(Exception ex, string customMessage = null, bool showStackTrace = true) - { - MessageBox.Show(string.Concat(customMessage ?? "Something went wrong (CTRL+C to copy):", Environment.NewLine, showStackTrace ? ex.ToString() : ex.Message), ex.Message, MessageBoxButtons.OK, MessageBoxIcon.Error); - } - private async void ExportResults(FileType defaultFileType) { string filePath = null; @@ -52,6 +49,7 @@ private async void ExportResults(FileType defaultFileType) filePath = this.exportFileDialog.FileName; var selectedFileType = Path.GetExtension(filePath).Equals(FileType.XLS.GetExtension()) ? FileType.XLS : FileType.CSV; + var stopWatch = Stopwatch.StartNew(); loadingIcon = this.ShowLoadingIcon("Exporting Data"); if (selectedFileType == FileType.CSV) { @@ -64,7 +62,7 @@ private async void ExportResults(FileType defaultFileType) { MessageBox.Show($"the .xls file format supports a maximum of {MAX_XLS_COLUMN_COUNT} columns.\r\n\r\nPlease try another file format or reduce the amount of columns you are exporting. Your columns: {this.MainDataSource.Columns.Count}", "Too many columns", MessageBoxButtons.OK, MessageBoxIcon.Error); - + return; } @@ -82,6 +80,7 @@ private async void ExportResults(FileType defaultFileType) } else { + FileExportEvent.FireAndForget(selectedFileType, new FileInfo(filePath).Length, this.mainGridView.RowCount, this.mainGridView.ColumnCount, stopWatch.ElapsedMilliseconds); MessageBox.Show("Export successful!", "Success", MessageBoxButtons.OK, MessageBoxIcon.Information); } } @@ -237,7 +236,7 @@ private static void HandleAllFilesSkippedException(AllFilesSkippedException ex) { sb.AppendLine($"-{skippedFile.FileName}"); } - ShowError(new Exception(sb.ToString(), ex.SkippedFiles.FirstOrDefault()?.Exception)); + throw new Exception(sb.ToString(), ex.SkippedFiles.FirstOrDefault()?.Exception); } private static void HandleSomeFilesSkippedException(SomeFilesSkippedException ex) @@ -248,13 +247,13 @@ private static void HandleSomeFilesSkippedException(SomeFilesSkippedException ex { sb.AppendLine($"-{skippedFile.FileName}"); } - ShowError(new Exception(sb.ToString(), ex.SkippedFiles.FirstOrDefault()?.Exception)); + throw new Exception(sb.ToString(), ex.SkippedFiles.FirstOrDefault()?.Exception); } private static void HandleFileReadException(FileReadException ex) { - ShowError(new Exception($"Could not load parquet file.{Environment.NewLine}{Environment.NewLine}" + - $"If the problem persists please consider opening a bug ticket in the project repo: Help -> About{Environment.NewLine}", ex)); + throw new Exception($"Could not load parquet file.{Environment.NewLine}{Environment.NewLine}" + + $"If the problem persists please consider opening a bug ticket in the project repo: Help -> About{Environment.NewLine}", ex); } private static void HandleMultipleSchemasFoundException(MultipleSchemasFoundException ex) @@ -275,7 +274,7 @@ private static void HandleMultipleSchemasFoundException(MultipleSchemasFoundExce sb.AppendLine($" {schema.Fields.ElementAt(i).Name}"); } } - ShowError(new Exception(sb.ToString(), ex)); + throw new Exception(sb.ToString(), ex); } } } diff --git a/src/ParquetViewer/MainForm.ToolStripMenus.cs b/src/ParquetViewer/MainForm.ToolStripMenus.cs index 3bd0122..e00a915 100644 --- a/src/ParquetViewer/MainForm.ToolStripMenus.cs +++ b/src/ParquetViewer/MainForm.ToolStripMenus.cs @@ -1,8 +1,10 @@ -using ParquetViewer.Helpers; +using ParquetViewer.Analytics; +using ParquetViewer.Helpers; using System; using System.Data; using System.Diagnostics; using System.IO; +using System.Threading.Tasks; using System.Windows.Forms; namespace ParquetViewer @@ -13,6 +15,7 @@ public partial class MainForm private void newToolStripMenuItem_Click(object sender, EventArgs e) { + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.FileNew); this.OpenFileOrFolderPath = null; } @@ -22,13 +25,14 @@ private async void openToolStripMenuItem_Click(object sender, EventArgs e) { if (this.openParquetFileDialog.ShowDialog(this) == DialogResult.OK) { + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.FileOpen); await this.OpenNewFileOrFolder(this.openParquetFileDialog.FileName); } } - catch (Exception ex) + catch { this.OpenFileOrFolderPath = null; - ShowError(ex); + throw; } } @@ -38,164 +42,126 @@ private async void openFolderToolStripMenuItem_Click(object sender, EventArgs e) { if (this.openFolderDialog.ShowDialog(this) == DialogResult.OK) { + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.FolderOpen); await this.OpenNewFileOrFolder(this.openFolderDialog.SelectedPath); } } - catch (Exception ex) + catch { this.OpenFileOrFolderPath = null; - ShowError(ex); + throw; } } - private void saveAsToolStripMenuItem_Click(object sender, EventArgs e) - { - try - { - this.ExportResults(default); - } - catch (Exception ex) - { - ShowError(ex); - } - } + private void saveAsToolStripMenuItem_Click(object sender, EventArgs e) => this.ExportResults(default); - private void exitToolStripMenuItem_Click(object sender, EventArgs e) + private async void exitToolStripMenuItem_Click(object sender, EventArgs e) { + await new MenuBarClickEvent { Action = MenuBarClickEvent.ActionId.Exit }.Record(); this.Close(); } private async void changeFieldsMenuStripButton_Click(object sender, EventArgs e) { - try - { - await this.OpenFieldSelectionDialog(true); - } - catch (Exception ex) - { - ShowError(ex); - } + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.ChangeFields); + await this.OpenFieldSelectionDialog(true); } private void GetSQLCreateTableScriptToolStripMenuItem_Click(object sender, EventArgs e) { - try + var openFileOrFolderPath = this.OpenFileOrFolderPath; + if (openFileOrFolderPath?.EndsWith("/") == true) { - var openFileOrFolderPath = this.OpenFileOrFolderPath; - if (openFileOrFolderPath?.EndsWith("/") == true) - { - //trim trailing slash '/' - openFileOrFolderPath = openFileOrFolderPath[..^1]; - } + //trim trailing slash '/' + openFileOrFolderPath = openFileOrFolderPath[..^1]; + } - string tableName = Path.GetFileNameWithoutExtension(openFileOrFolderPath) ?? DEFAULT_TABLE_NAME; - if (this.mainDataSource?.Columns.Count > 0) - { - var dataset = new DataSet(); + string tableName = Path.GetFileNameWithoutExtension(openFileOrFolderPath) ?? DEFAULT_TABLE_NAME; + if (this.mainDataSource?.Columns.Count > 0) + { + var dataset = new DataSet(); - this.mainDataSource.TableName = tableName; - dataset.Tables.Add(this.mainDataSource); + this.mainDataSource.TableName = tableName; + dataset.Tables.Add(this.mainDataSource); - var scriptAdapter = new CustomScriptBasedSchemaAdapter(); - string sql = scriptAdapter.GetSchemaScript(dataset, false); + var scriptAdapter = new CustomScriptBasedSchemaAdapter(); + string sql = scriptAdapter.GetSchemaScript(dataset, false); - Clipboard.SetText(sql); - MessageBox.Show(this, "Create table script copied to clipboard!", "Parquet Viewer", MessageBoxButtons.OK, MessageBoxIcon.Information); - } - else - MessageBox.Show(this, "Please select some fields first to get the SQL script", "Parquet Viewer", MessageBoxButtons.OK, MessageBoxIcon.Error); - } - catch (Exception ex) - { - ShowError(ex); + Clipboard.SetText(sql); + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.SQLCreateTable); + MessageBox.Show(this, "Create table script copied to clipboard!", "Parquet Viewer", MessageBoxButtons.OK, MessageBoxIcon.Information); } + else + MessageBox.Show(this, "Please select some fields first to get the SQL script", "Parquet Viewer", MessageBoxButtons.OK, MessageBoxIcon.Error); } private void MetadataViewerToolStripMenuItem_Click(object sender, EventArgs e) { - try + if (IsAnyFileOpen) { - if (IsAnyFileOpen) - { - using var metadataViewer = new MetadataViewer(this._openParquetEngine); - metadataViewer.ShowDialog(this); - } - } - catch (Exception ex) - { - ShowError(ex); + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.MetadataViewer); + using var metadataViewer = new MetadataViewer(this._openParquetEngine); + metadataViewer.ShowDialog(this); } } private void changeColumnSizingToolStripMenuItem_Click(object sender, EventArgs e) { - try + if (sender is ToolStripMenuItem tsi && tsi.Tag != null + && Enum.TryParse(tsi.Tag.ToString(), out AutoSizeColumnsMode columnSizingMode) + && AppSettings.AutoSizeColumnsMode != columnSizingMode) { - if (sender is ToolStripMenuItem tsi && tsi.Tag != null - && Enum.TryParse(tsi.Tag.ToString(), out AutoSizeColumnsMode columnSizingMode) - && AppSettings.AutoSizeColumnsMode != columnSizingMode) + AppSettings.AutoSizeColumnsMode = columnSizingMode; + foreach (ToolStripMenuItem toolStripItem in tsi.GetCurrentParent().Items) { - AppSettings.AutoSizeColumnsMode = columnSizingMode; - foreach (ToolStripMenuItem toolStripItem in tsi.GetCurrentParent().Items) - { - toolStripItem.Checked = toolStripItem.Tag?.Equals(tsi.Tag) == true; - } - this.mainGridView.AutoSizeColumns(); - - //Also clear out each column's Tag so auto sizing can pick it up again (see: FastAutoSizeColumns()) - foreach (DataGridViewColumn column in this.mainGridView.Columns) - { - column.Tag = null; //TODO: This logic is terrible. Need to find a cleaner solution - } + toolStripItem.Checked = toolStripItem.Tag?.Equals(tsi.Tag) == true; + } + this.mainGridView.AutoSizeColumns(); + + //Also clear out each column's Tag so auto sizing can pick it up again (see: FastAutoSizeColumns()) + foreach (DataGridViewColumn column in this.mainGridView.Columns) + { + column.Tag = null; //TODO: This logic is terrible. Need to find a cleaner solution } - } - catch (Exception ex) - { - ShowError(ex); } } private void rememberRecordCountToolStripMenuItem_Click(object sender, EventArgs e) { - try - { - this.rememberRecordCountToolStripMenuItem.Checked = !this.rememberRecordCountToolStripMenuItem.Checked; - AppSettings.RememberLastRowCount = this.rememberRecordCountToolStripMenuItem.Checked; - AppSettings.LastRowCount = this.CurrentMaxRowCount; - } - catch (Exception ex) - { - ShowError(ex); - } + this.rememberRecordCountToolStripMenuItem.Checked = !this.rememberRecordCountToolStripMenuItem.Checked; + AppSettings.RememberLastRowCount = this.rememberRecordCountToolStripMenuItem.Checked; + AppSettings.LastRowCount = this.CurrentMaxRowCount; } private void aboutToolStripMenuItem_Click(object sender, EventArgs e) { + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.AboutBox); (new AboutBox()).ShowDialog(this); } private void userGuideToolStripMenuItem_Click(object sender, EventArgs e) { + MenuBarClickEvent.FireAndForget(MenuBarClickEvent.ActionId.UserGuide); Process.Start(new ProcessStartInfo(Constants.WikiURL) { UseShellExecute = true }); } private void DateFormatMenuItem_Click(object sender, EventArgs e) { - try + if (sender is ToolStripMenuItem item) { - if (sender is ToolStripMenuItem item) - { - var selectedDateFormat = (DateFormat)(int.Parse((string)item.Tag)); - AppSettings.DateTimeDisplayFormat = selectedDateFormat; - this.RefreshDateFormatMenuItemSelection(); - this.mainGridView.UpdateDateFormats(); - this.mainGridView.Refresh(); - } - } - catch (Exception ex) - { - ShowError(ex); + var selectedDateFormat = (DateFormat)(int.Parse((string)item.Tag)); + AppSettings.DateTimeDisplayFormat = selectedDateFormat; + this.RefreshDateFormatMenuItemSelection(); + this.mainGridView.UpdateDateFormats(); + this.mainGridView.Refresh(); } } + + private void shareAnonymousUsageDataToolStripMenuItem_Click(object sender, EventArgs e) + { + this.shareAnonymousUsageDataToolStripMenuItem.Checked = !this.shareAnonymousUsageDataToolStripMenuItem.Checked; + AppSettings.AnalyticsDataGatheringConsent = this.shareAnonymousUsageDataToolStripMenuItem.Checked; + AppSettings.ConsentLastAskedOnVersion = AboutBox.AssemblyVersion; + } } } diff --git a/src/ParquetViewer/MainForm.cs b/src/ParquetViewer/MainForm.cs index 68dc990..9489239 100644 --- a/src/ParquetViewer/MainForm.cs +++ b/src/ParquetViewer/MainForm.cs @@ -1,5 +1,7 @@ +using ParquetViewer.Analytics; using ParquetViewer.Engine; using ParquetViewer.Engine.Exceptions; +using ParquetViewer.Exceptions; using ParquetViewer.Helpers; using System; using System.Collections.Concurrent; @@ -8,7 +10,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Windows.Forms; @@ -18,7 +19,6 @@ public partial class MainForm : Form { private const int DefaultOffset = 0; private const int DefaultRowCountValue = 1000; - private const int PerformanceWarningCellCount = int.MaxValue; //Disabling this for now because it might not be needed anymore now that we have fast auto column sizing private const int MultiThreadedParquetEngineColumnCountThreshold = 1000; private readonly string DefaultFormTitle; @@ -127,25 +127,6 @@ private DataTable MainDataSource get => this.mainDataSource; set { - //Check for performance issues - int? cellsToRender = value?.Columns.Count * value?.Rows.Count; - if (cellsToRender > PerformanceWarningCellCount && AppSettings.AutoSizeColumnsMode == AutoSizeColumnsMode.AllCells) - { - //Don't spam the user so ask only once per app update - if (AppSettings.WarningBypassedOnVersion != AboutBox.AssemblyVersion) - { - var choice = MessageBox.Show(this, $"Looks like you're loading a lot of data with column sizing set to 'Fit Headers & Content'. This might cause significant load times. " + - Environment.NewLine + Environment.NewLine + $"If you experience performance issues try changing the default column sizing: Edit -> Column Sizing" + - Environment.NewLine + Environment.NewLine + "Continue loading file anyway?", "Performance Warning", - MessageBoxButtons.OKCancel); - - if (choice == DialogResult.Cancel) - return; - else - AppSettings.WarningBypassedOnVersion = AboutBox.AssemblyVersion; - } - } - this.mainDataSource = value; this.mainGridView.DataSource = this.mainDataSource; } @@ -175,35 +156,32 @@ public MainForm(string fileToOpenPath) : this() private void MainForm_Load(object sender, EventArgs e) { - try + //Open existing file on first load (Usually this means user "double clicked" a parquet file with this utility as the default program). + if (!string.IsNullOrWhiteSpace(this.fileToLoadOnLaunch)) { - //Open existing file on first load (Usually this means user "double clicked" a parquet file with this utility as the default program). - if (!string.IsNullOrWhiteSpace(this.fileToLoadOnLaunch)) - { - this.OpenNewFileOrFolder(this.fileToLoadOnLaunch); - } + this.OpenNewFileOrFolder(this.fileToLoadOnLaunch); + } - //Setup date format checkboxes - this.RefreshDateFormatMenuItemSelection(); + //Setup date format checkboxes + this.RefreshDateFormatMenuItemSelection(); - foreach (ToolStripMenuItem toolStripItem in this.columnSizingToolStripMenuItem.DropDown.Items) + foreach (ToolStripMenuItem toolStripItem in this.columnSizingToolStripMenuItem.DropDown.Items) + { + if (toolStripItem.Tag?.Equals(AppSettings.AutoSizeColumnsMode.ToString()) == true) { - if (toolStripItem.Tag?.Equals(AppSettings.AutoSizeColumnsMode.ToString()) == true) - { - toolStripItem.Checked = true; - break; - } + toolStripItem.Checked = true; + break; } - - if (AppSettings.RememberLastRowCount) - this.rememberRecordCountToolStripMenuItem.Checked = true; - else - this.rememberRecordCountToolStripMenuItem.Checked = false; - } - catch (Exception ex) - { - ShowError(ex); } + + if (AppSettings.RememberLastRowCount) + this.rememberRecordCountToolStripMenuItem.Checked = true; + else + this.rememberRecordCountToolStripMenuItem.Checked = false; + + //Get user's consent to gather analytics; and update the toolstrip menu item accordingly + Program.GetUserConsentToGatherAnalytics(); + this.shareAnonymousUsageDataToolStripMenuItem.Checked = AppSettings.AnalyticsDataGatheringConsent; } private async Task OpenFieldSelectionDialog(bool forceOpenDialog) @@ -355,10 +333,13 @@ await Parallel.ForEachAsync(fieldGroups, options, }, loadingIcon.CancellationToken); this.recordCountStatusBarLabel.Text = string.Format("{0} to {1}", this.CurrentOffset, this.CurrentOffset + finalResult.Rows.Count); - this.totalRowCountStatusBarLabel.Text = finalResult.ExtendedProperties[ParquetViewer.Engine.ParquetEngine.TotalRecordCountExtendedPropertyKey].ToString(); + this.totalRowCountStatusBarLabel.Text = finalResult.ExtendedProperties[Engine.ParquetEngine.TotalRecordCountExtendedPropertyKey].ToString(); this.actualShownRecordCountLabel.Text = finalResult.Rows.Count.ToString(); this.MainDataSource = finalResult; + + FileOpenEvent.FireAndForget(Directory.Exists(this.OpenFileOrFolderPath), this._openParquetEngine.NumberOfPartitions, this._openParquetEngine.RecordCount, this._openParquetEngine.ThriftMetadata.RowGroups.Count(), + this._openParquetEngine.Fields.Count(), finalResult.Columns.Cast().Select(column => column.DataType.Name).Distinct().Order().ToArray(), this.CurrentOffset, this.CurrentMaxRowCount, finalResult.Columns.Count, stopwatch.ElapsedMilliseconds); } } catch (AllFilesSkippedException ex) @@ -380,7 +361,7 @@ await Parallel.ForEachAsync(fieldGroups, options, catch (Exception ex) { if (ex is not OperationCanceledException) - ShowError(ex); + throw; } finally { @@ -409,10 +390,10 @@ private void runQueryButton_Click(object sender, EventArgs e) if (this.IsAnyFileOpen) { string queryText = this.searchFilterTextBox.Text ?? string.Empty; - queryText = Regex.Replace(queryText, QueryUselessPartRegex, string.Empty).Trim(); + queryText = QueryUselessPartRegex().Replace(queryText, string.Empty).Trim(); //Treat list and map types as strings by casting them automatically - foreach(var complexField in this.mainGridView.Columns.OfType() + foreach (var complexField in this.mainGridView.Columns.OfType() .Where(c => c.ValueType == typeof(ListValue) || c.ValueType == typeof(MapValue)) .Select(c => c.Name)) { @@ -420,14 +401,27 @@ private void runQueryButton_Click(object sender, EventArgs e) queryText = queryText.Replace(complexField, $"CONVERT({complexField}, System.String)"); } + var queryEvent = new ExecuteQueryEvent + { + RecordCount = this.MainDataSource.Rows.Count, + ColumnCount = this.MainDataSource.Columns.Count + }; + var stopwatch = Stopwatch.StartNew(); + try { this.MainDataSource.DefaultView.RowFilter = queryText; + queryEvent.IsValid = true; } catch (Exception ex) { this.MainDataSource.DefaultView.RowFilter = null; - ShowError(ex, "The query doesn't seem to be valid. Please try again.", false); + throw new InvalidQueryException(ex); + } + finally + { + queryEvent.RunTimeMS = stopwatch.ElapsedMilliseconds; + var _ = queryEvent.Record(); //Fire and forget } } } diff --git a/src/ParquetViewer/MainForm.resx b/src/ParquetViewer/MainForm.resx index 422a78b..17c2ae1 100644 --- a/src/ParquetViewer/MainForm.resx +++ b/src/ParquetViewer/MainForm.resx @@ -1,4 +1,64 @@ - + + + diff --git a/src/ParquetViewer/MetadataViewer.cs b/src/ParquetViewer/MetadataViewer.cs index ed796b1..20e55d3 100644 --- a/src/ParquetViewer/MetadataViewer.cs +++ b/src/ParquetViewer/MetadataViewer.cs @@ -11,9 +11,9 @@ public partial class MetadataViewer : Form private static readonly string THRIFT_METADATA = "Thrift Metadata"; private static readonly string APACHE_ARROW_SCHEMA = "ARROW:schema"; private static readonly string PANDAS_SCHEMA = "pandas"; - private ParquetViewer.Engine.ParquetEngine parquetEngine; + private Engine.ParquetEngine parquetEngine; - public MetadataViewer(ParquetViewer.Engine.ParquetEngine parquetEngine) : this() + public MetadataViewer(Engine.ParquetEngine parquetEngine) : this() { this.parquetEngine = parquetEngine; } @@ -56,8 +56,7 @@ private void MainBackgroundWorker_DoWork(object sender, System.ComponentModel.Do var metadataResult = new List<(string TabName, string Text)>(); if (parquetEngine.ThriftMetadata != null) { - - string json = ParquetMetadataAnalyzers.ThriftMetadataToJSON(parquetEngine.ThriftMetadata, parquetEngine.RecordCount, parquetEngine.Fields.Count); + string json = ParquetMetadataAnalyzers.ThriftMetadataToJSON(parquetEngine, parquetEngine.RecordCount, parquetEngine.Fields.Count); metadataResult.Add((THRIFT_METADATA, json)); } else diff --git a/src/ParquetViewer/ParquetViewer.csproj b/src/ParquetViewer/ParquetViewer.csproj index 39aedf6..0aa6491 100644 --- a/src/ParquetViewer/ParquetViewer.csproj +++ b/src/ParquetViewer/ParquetViewer.csproj @@ -35,7 +35,6 @@ - diff --git a/src/ParquetViewer/Program.cs b/src/ParquetViewer/Program.cs index d8c9157..2d87633 100644 --- a/src/ParquetViewer/Program.cs +++ b/src/ParquetViewer/Program.cs @@ -1,7 +1,10 @@ -using System; +using ParquetViewer.Analytics; +using System; using System.IO; using System.Windows.Forms; +#nullable enable + namespace ParquetViewer { static class Program @@ -10,9 +13,9 @@ static class Program /// The main entry point for the application. /// [STAThread] - static void Main(string[] args) + private static void Main(string[] args) { - string fileToOpen = null; + string? fileToOpen = null; try { if (args?.Length > 0 && File.Exists(args[0])) @@ -27,12 +30,65 @@ static void Main(string[] args) //Form must be created after calling SetCompatibleTextRenderingDefault(); Form mainForm; - if (string.IsNullOrWhiteSpace(fileToOpen)) - mainForm = new MainForm(); - else + bool isOpeningFile = !string.IsNullOrWhiteSpace(fileToOpen); + if (isOpeningFile) mainForm = new MainForm(fileToOpen); + else + mainForm = new MainForm(); + + RouteUnhandledExceptions(); + ProgramOpenEvent.FireAndForget(isOpeningFile); Application.Run(mainForm); } + + /// + /// When called, all unhandled exceptions within the runtime and winforms UI thread + /// will be routed to the handler. + /// + /// Side effect: The application will never quit when an unhandled exception happens. + private static void RouteUnhandledExceptions() + { + //If we're not debugging, route all unhandled exceptions to our top level exception handler + if (!System.Diagnostics.Debugger.IsAttached) + { + // Add the event handler for handling non-UI thread exceptions to the event. + AppDomain.CurrentDomain.UnhandledException += new ((sender, e) => ExceptionHandler((Exception)e.ExceptionObject)); + + // Add the event handler for handling UI thread exceptions to the event. + Application.ThreadException += new ((sender, e) => ExceptionHandler(e.Exception)); + } + } + + private static void ExceptionHandler(Exception ex) + { + ExceptionEvent.FireAndForget(ex); + MessageBox.Show($"Something went wrong (CTRL+C to copy):{Environment.NewLine}{ex}", ex.Message, MessageBoxButtons.OK, MessageBoxIcon.Error); + } + + public static void GetUserConsentToGatherAnalytics() + { + if (!AppSettings.AnalyticsDataGatheringConsent && AssemblyVersionToInt(AppSettings.ConsentLastAskedOnVersion) < AssemblyVersionToInt(AboutBox.AssemblyVersion)) + { + bool isFirstLaunch = AppSettings.ConsentLastAskedOnVersion is null; + if (isFirstLaunch) + { + //Don't ask for consent on the first launch. Lets do it on the second one so its less annoying. + AppSettings.ConsentLastAskedOnVersion = "0"; + } + else + { + AppSettings.ConsentLastAskedOnVersion = AboutBox.AssemblyVersion; + if (MessageBox.Show($"Would you like to share anonymous usage data to help make ParquetViewer better?{Environment.NewLine}{Environment.NewLine}" + + $"You can always change this setting later from the Help menu.", "Share Anonymous Usage Data?", MessageBoxButtons.YesNo, MessageBoxIcon.Question) == DialogResult.Yes) + { + //We got consent! Start gathering some data.. + AppSettings.AnalyticsDataGatheringConsent = true; + } + } + } + + int AssemblyVersionToInt(string? version) => int.Parse(version?.Replace(".", string.Empty) ?? "0"); + } } } diff --git a/src/ParquetViewer/Properties/AssemblyInfo.cs b/src/ParquetViewer/Properties/AssemblyInfo.cs index c7fd995..7c8548d 100644 --- a/src/ParquetViewer/Properties/AssemblyInfo.cs +++ b/src/ParquetViewer/Properties/AssemblyInfo.cs @@ -33,4 +33,4 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.7.0.3")] +[assembly: AssemblyVersion("2.7.1.0")] From f4826dd0ce9e4cd63a57a3f543641c2231a4214b Mon Sep 17 00:00:00 2001 From: Sal Date: Mon, 29 May 2023 22:15:13 -0400 Subject: [PATCH 2/3] add better unknown support --- src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs b/src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs index 843cbbf..f5d70d7 100644 --- a/src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs +++ b/src/ParquetViewer/Helpers/ParquetMetadataAnalyzers.cs @@ -150,6 +150,10 @@ object ProcessSchemaTree(Engine.ParquetSchemaElement parquetSchemaElement) { return new { Name = nameof(logicalType.UUID) }; } + else if (logicalType.UNKNOWN is not null) + { + return new { Name = $"{logicalType.UNKNOWN.GetType().Name}" }; + } else { return new { Name = nameof(logicalType.UNKNOWN) }; From 8bba3638127693634e789ffa0596c9f50e8e3cb7 Mon Sep 17 00:00:00 2001 From: Sal Date: Mon, 29 May 2023 22:24:03 -0400 Subject: [PATCH 3/3] fix tests --- src/ParquetViewer.Tests/TestAmplitudeEvent.cs | 1 + src/ParquetViewer/Analytics/AmplitudeEvent.cs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ParquetViewer.Tests/TestAmplitudeEvent.cs b/src/ParquetViewer.Tests/TestAmplitudeEvent.cs index 1be63e4..40669cd 100644 --- a/src/ParquetViewer.Tests/TestAmplitudeEvent.cs +++ b/src/ParquetViewer.Tests/TestAmplitudeEvent.cs @@ -20,6 +20,7 @@ public TestAmplitudeEvent(string dummyApiKey) : base(EVENT_TYPE) public void SwapHttpClientHandler(HttpMessageHandler mockHandler) { HttpMessageHandler = mockHandler; + BypassConsentRequirement = true; } } } diff --git a/src/ParquetViewer/Analytics/AmplitudeEvent.cs b/src/ParquetViewer/Analytics/AmplitudeEvent.cs index 985f862..08f2897 100644 --- a/src/ParquetViewer/Analytics/AmplitudeEvent.cs +++ b/src/ParquetViewer/Analytics/AmplitudeEvent.cs @@ -17,6 +17,7 @@ public abstract class AmplitudeEvent private static readonly int _systemRAM = (int)(GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 1048576.0 /*magic number*/); protected HttpMessageHandler HttpMessageHandler { get; set; } = new HttpClientHandler(); + protected bool BypassConsentRequirement { get; set; } [JsonIgnore] public string DeviceId => AppSettings.AnalyticsDeviceId.ToString(); @@ -48,7 +49,7 @@ public async Task Record() { try { - if (AMPLITUDE_API_KEY.Length == 0 || !AppSettings.AnalyticsDataGatheringConsent) + if (!BypassConsentRequirement && (AMPLITUDE_API_KEY.Length == 0 || !AppSettings.AnalyticsDataGatheringConsent)) return false; var request = new