Merge pull request #148 from tableau/Feb2022

publish latest logshark changes to github

.gitignore

@@ -19,8 +19,10 @@ Temp/
 
 # Test data
 TestLogs/
-Properties/
 .terraform
 *.zip
-!/**/TestData/**/*.zip
 .vscode/
+
+# code coverage
+coverage.json
+coverage.cobertura.xml

LogShark.Shared/Extensions/StringExtensions.cs

@@ -71,24 +71,28 @@ public static string GetWorkerIdFromFilePath(this string fullPath)
             return "worker0"; // Tabadmin primary or Desktop
         }
 
-        public static Match GetRegexMatchAndMoveCorrectRegexUpFront(this string lineToMatch, IList<Regex> regexList)
+        // Lock is needed because this logic can reshuffle the list. This could cause race conditions between threads.
+        public static Match GetRegexMatchAndMoveCorrectRegexUpFront(this string lineToMatch, IList<Regex> regexList, object regexListLockObject)
         {
             if (regexList == null)
             {
                 return null;
             }
-            
-            for (var i = 0; i < regexList.Count; ++i)
+
+            lock (regexListLockObject)
             {
-                var match = regexList[i].Match(lineToMatch);
-                if (match.Success)
+                for (var i = 0; i < regexList.Count; ++i)
                 {
-                    regexList.MoveToFront(i);
-                    return match;
+                    var match = regexList[i].Match(lineToMatch);
+                    if (match.Success)
+                    {
+                        regexList.MoveToFront(i);
+                        return match;
+                    }
                 }
+
+                return null;
             }
-
-            return null;
         }
 
         public static string TrimSurroundingDoubleQuotes(this string original)

LogShark.Shared/LogReading/Containers/LogSetInfo.cs

@@ -1,18 +1,23 @@
+using System.Collections.Generic;
+
 namespace LogShark.Shared.LogReading.Containers
 {
     public class LogSetInfo
     {
+        public List<string> FilePaths { get; }
         public bool IsZip { get; }
         public string Path { get; }
         public string Prefix { get; }
         public string RootPath { get; }
 
-        public LogSetInfo(string path, string prefix, bool isZip, string rootPath)
+        public LogSetInfo(List<string> filePaths, string path, string prefix, bool isZip, string rootPath)
         {
+            FilePaths = filePaths;
             IsZip = isZip;
             Path = path;
             Prefix = prefix;
             RootPath = rootPath;
         }
     }
-}
+}
+

LogShark.Shared/LogReading/TableauLogsExtractor.cs

@@ -71,7 +71,7 @@ public static FileCanBeOpenedResult FileCanBeOpened(string path, ILogger logger)
                     var reader = new SimpleLinePerLineReader(stream);
                     var line = reader.ReadLines().FirstOrDefault(); // This would fail if zip is password protected 
                 }
-                    
+
                 return FileCanBeOpenedResult.Success();
             }
             catch (Exception ex)
@@ -134,11 +134,12 @@ private IList<LogSetInfo> EvaluateLogSetStructure(string logSetPath, string temp
             IsDirectory = IsPathADirectory(logSetPath);
             _logger.LogInformation("Provided log set appears to be {logSetType}", IsDirectory ? "directory" : "zip file");
 
-            var rootSetInfo = new LogSetInfo(logSetPath, string.Empty, !IsDirectory, _rootPath);
+            var rootFilePaths = GetFilePaths(logSetPath, IsDirectory);
+            var rootSetInfo = new LogSetInfo(rootFilePaths, logSetPath, string.Empty, !IsDirectory, _rootPath);
             var zipInfoList = new List<LogSetInfo> {rootSetInfo};
 
             zipInfoList.AddRange(IsDirectory
-                ? LookForZippedParts(logSetPath)
+                ? LookForZippedPartsInDirectory(logSetPath)
                 : HandleNestedZips(logSetPath, tempDirRoot, processingNotificationsCollector)
             );
 
@@ -178,7 +179,8 @@ private IEnumerable<LogSetInfo> HandleNestedZips(string zippedLogSetPath, string
                 var checkResult = FileCanBeOpened(extractPath, _logger); 
                 if (checkResult.FileCanBeOpened)
                 {
-                    nestedZipFilesInfo.Add(new LogSetInfo(extractPath.NormalizeSeparatorsToUnix(), nestedZip.FullName.RemoveZipFromTail(), true, _rootPath));
+                    var filePaths = GetFilePaths(extractPath, false);
+                    nestedZipFilesInfo.Add(new LogSetInfo(filePaths, extractPath.NormalizeSeparatorsToUnix(), nestedZip.FullName.RemoveZipFromTail(), true, _rootPath));
                 }
                 else
                 {
@@ -191,14 +193,18 @@ private IEnumerable<LogSetInfo> HandleNestedZips(string zippedLogSetPath, string
             return nestedZipFilesInfo;
         }
 
-        private IEnumerable<LogSetInfo> LookForZippedParts(string logSetDirectoryPath)
+        private IEnumerable<LogSetInfo> LookForZippedPartsInDirectory(string logSetDirectoryPath)
         {
             _logger.LogInformation("Looking for typical zip files inside '{logSetPath}' folder", logSetDirectoryPath);
 
             var allFiles = Directory.EnumerateFiles(logSetDirectoryPath, "*", SearchOption.AllDirectories);
             var allKnownZips = allFiles
                 .Where(path => IsNestedZip(path, true))
-                .Select(path => new LogSetInfo(path.NormalizeSeparatorsToUnix(), path.NormalizePath(_rootPath).RemoveZipFromTail(), true, _rootPath))
+                .Select(path =>
+                {
+                    var filePaths = GetFilePaths(path, false);
+                    return new LogSetInfo(filePaths, path.NormalizeSeparatorsToUnix(), path.NormalizePath(_rootPath).RemoveZipFromTail(), true, _rootPath);
+                })
                 .ToList();
 
             if (allKnownZips.Count > 0)
@@ -233,5 +239,20 @@ private static long GetDirSize(string path)
                 .GetFiles("*", SearchOption.AllDirectories)
                 .Sum(fileInfo => fileInfo.Length);
         }
+
+        private static List<string> GetFilePaths(string path, bool isDirectory)
+        {
+            if (isDirectory)
+            {
+                return Directory
+                    .EnumerateFiles(path, "*", SearchOption.AllDirectories)
+                    .ToList();
+            }
+
+            using var zipArchive = ZipFile.Open(path, ZipArchiveMode.Read);
+            return zipArchive.Entries
+                .Select(entry => entry.FullName)
+                .ToList();
+        }
     }
 }

LogShark.Shared/LogType.cs

@@ -26,6 +26,7 @@ public enum LogType
         VizportalJava,
         VizqlserverCpp,
         VizqlDesktop,
+        UnknownLogTypeForErrors,
         WorkgroupYml,
         Zookeeper
     }

LogShark.Tests/TestData/EndToEndTests/Expected/logs_clean_tsm_csv/csv/TabadminController_TabadminControllerBuildRecords.csv

@@ -0,0 +1,29 @@
+RoundedTimestamp,Build
+08/09/2019 22:00:00 +00:00,20192.19.0718.1543
+08/09/2019 20:24:00 +00:00,20192.19.0718.1543
+08/09/2019 20:43:00 +00:00,20192.19.0718.1543
+08/09/2019 21:02:00 +00:00,20192.19.0718.1543
+08/09/2019 21:40:00 +00:00,20192.19.0718.1543
+08/09/2019 20:33:00 +00:00,20192.19.0718.1543
+08/09/2019 21:59:00 +00:00,20192.19.0718.1543
+08/09/2019 21:30:00 +00:00,20192.19.0718.1543
+08/09/2019 20:23:00 +00:00,20192.19.0718.1543
+08/09/2019 20:42:00 +00:00,20192.19.0718.1543
+08/09/2019 20:32:00 +00:00,20192.19.0718.1543
+08/09/2019 21:58:00 +00:00,20192.19.0718.1543
+08/09/2019 21:29:00 +00:00,20192.19.0718.1543
+08/09/2019 20:22:00 +00:00,20192.19.0718.1543
+08/09/2019 20:41:00 +00:00,20192.19.0718.1543
+08/09/2019 20:31:00 +00:00,20192.19.0718.1543
+08/09/2019 21:28:00 +00:00,20192.19.0718.1543
+08/09/2019 20:40:00 +00:00,20192.19.0718.1543
+08/09/2019 20:39:00 +00:00,20192.19.0718.1543
+08/09/2019 18:43:00 +00:00,20192.19.0718.1543
+08/09/2019 18:42:00 +00:00,20192.19.0718.1543
+08/09/2019 21:05:00 +00:00,20192.19.0718.1543
+08/09/2019 18:41:00 +00:00,20192.19.0718.1543
+08/09/2019 21:04:00 +00:00,20192.19.0718.1543
+08/09/2019 21:42:00 +00:00,20192.19.0718.1543
+08/09/2019 20:25:00 +00:00,20192.19.0718.1543
+08/09/2019 21:03:00 +00:00,20192.19.0718.1543
+08/09/2019 21:41:00 +00:00,20192.19.0718.1543

LogShark/Config/LogSharkConfig.json

@@ -13,6 +13,7 @@
     }
   },
   "EnvironmentConfig": {
+    "NumberOfProcessingThreads": 2,
     "TempDirOverride": null,
     "OutputDir": "Output",
     "OutputDirMaxResults": null,
@@ -60,7 +61,8 @@
       "Id": "",
       "Name": ""
     },
-    "ProjectDescriptionFooterHtml": ""
+    "ProjectDescriptionFooterHtml": "",
+    "SkipPublishingEmptyWorkbooks": true
   },
   "PluginsConfiguration": {
     "DefaultPluginSet": {
@@ -85,5 +87,5 @@
     "VizqlDesktop": {
       "MaxQueryLength": 10000
     }
-  }
+  }    
 }

LogShark/Containers/ProcessFileResult.cs

@@ -1,17 +1,36 @@
+using System;
+using LogShark.Shared;
+
 namespace LogShark.Containers
 {
     public class ProcessFileResult
     {
+        public TimeSpan Elapsed { get; }
         public string ErrorMessage { get; }
         public ExitReason ExitReason { get; }
+        public long FileSizeBytes { get; }
         public long LinesProcessed { get; }
+        public LogType LogType { get; }
         public bool IsSuccessful => ExitReason == ExitReason.CompletedSuccessfully;
 
-        public ProcessFileResult(long linesProcessed, string errorMessage = null, ExitReason exitReason = ExitReason.CompletedSuccessfully)
+        public ProcessFileResult(LogType logType, ProcessStreamResult processStreamResult, long fileSizeBytes, TimeSpan elapsed)
+        {
+            Elapsed = elapsed;
+            ErrorMessage = processStreamResult.ErrorMessage;
+            ExitReason = processStreamResult.ExitReason;
+            FileSizeBytes = fileSizeBytes;
+            LinesProcessed = processStreamResult.LinesProcessed;
+            LogType = logType;
+        }
+
+        public ProcessFileResult(LogType logType, string errorMessage, ExitReason exitReason, long linesProcessed = 0)
         {
+            Elapsed = TimeSpan.Zero;
             ErrorMessage = errorMessage;
             ExitReason = exitReason;
+            FileSizeBytes = 0;
             LinesProcessed = linesProcessed;
+            LogType = logType;
         }
     }
 }

LogShark/Containers/ProcessLogTypeResult.cs

@@ -23,14 +23,9 @@ public ProcessLogTypeResult()
             LinesProcessed = 0;
         }
 
-        public void AddProcessingInfo(TimeSpan elapsed, long filesSizeBytes, ProcessFileResult processFileResult, int filesProcessed = 1)
+        public void AddProcessFileResults(ProcessFileResult processFileResult)
         {
-            AddProcessingInfo(elapsed, filesSizeBytes, processFileResult.LinesProcessed, filesProcessed, processFileResult.ErrorMessage, processFileResult.ExitReason);
-        }
-
-        public void AddNumbersFrom(ProcessLogTypeResult otherResult)
-        {
-            AddProcessingInfo(otherResult.Elapsed, otherResult.FilesSizeBytes, otherResult.LinesProcessed, otherResult.FilesProcessed, otherResult.ErrorMessage, otherResult.ExitReason);
+            AddProcessingInfo(processFileResult.Elapsed, processFileResult.FileSizeBytes, processFileResult.LinesProcessed, 1, processFileResult.ErrorMessage, processFileResult.ExitReason);
         }
 
         public override string ToString()
@@ -39,29 +34,28 @@ public override string ToString()
             var mbPerSecond = Elapsed.TotalSeconds > 0
                 ? fileSizeMb / Elapsed.TotalSeconds
                 : fileSizeMb;
-            
+
             return $"Processed {FilesProcessed} files in {Elapsed}. Elapsed: {Elapsed}. Total size: {fileSizeMb} MB. Processing at {mbPerSecond:0.00} MB/sec";
         }
-        
+
         private void AddProcessingInfo(TimeSpan elapsed, long filesSizeBytes, long linesProcessed, int filesProcessed, string errorMessage, ExitReason exitReason)
         {
-            Elapsed += elapsed;
+            Elapsed = new TimeSpan(Math.Max(Elapsed.Ticks, elapsed.Ticks)); // Take the larger of the two timespans since log parts are processed in parallel
             FilesSizeBytes += filesSizeBytes;
             LinesProcessed += linesProcessed;
             FilesProcessed += filesProcessed;
 
-            if (errorMessage != null)
+            if (errorMessage != null && exitReason != ExitReason.TaskCancelled)
             {
                 ErrorMessage = ErrorMessage == null
                     ? errorMessage
-                    : throw new LogSharkProgramLogicException($"{nameof(ProcessLogTypeResult)} already contains error and code is trying to push in another one. Why did not we stop at the first error? Previous error: `{ErrorMessage}`. New error: `{errorMessage}`");
+                    : ErrorMessage + $"\n\nAdditional error message from another thread: {errorMessage}"; 
             }
 
-            if (exitReason != ExitReason.CompletedSuccessfully)
+            if (exitReason != ExitReason.CompletedSuccessfully && exitReason != ExitReason.TaskCancelled)
             {
-                ExitReason = ExitReason == ExitReason.CompletedSuccessfully
-                    ? exitReason
-                    : throw new LogSharkConfigurationException($"{nameof(ProcessLogTypeResult)} already contains non-successful exit reason and code is trying to push in another one. Why did not we stop at the first error? Previous exit reason: `{ExitReason}`. New exit reason: `{exitReason}`");
+                ExitReason = ExitReason == ExitReason.CompletedSuccessfully || ExitReason == exitReason
+                    ? exitReason : ExitReason.MultipleExitReasonsOnDifferentThreads;
             }
         }
     }

LogShark/Containers/ProcessStreamResult.cs

@@ -0,0 +1,16 @@
+namespace LogShark.Containers
+{
+    public class ProcessStreamResult
+    {
+        public string ErrorMessage { get; }
+        public ExitReason ExitReason { get; }
+        public long LinesProcessed { get; }
+
+        public ProcessStreamResult(long linesProcessed, string errorMessage = null, ExitReason exitReason = ExitReason.CompletedSuccessfully)
+        {
+            ErrorMessage = errorMessage;
+            ExitReason = exitReason;
+            LinesProcessed = linesProcessed;
+        }
+    }
+}

LogShark/Containers/RunSummary.cs

@@ -82,11 +82,13 @@ private RunSummary(
             {
                 case ExitReason.BadLogSet:
                 case ExitReason.IncorrectConfiguration:
+                case ExitReason.LogLineTooLong:
                 case ExitReason.LogSetDoesNotContainRelevantLogs:
                 case ExitReason.OutOfMemory:
                     return false;
                 case ExitReason.CompletedSuccessfully:
                 case ExitReason.UnclassifiedError:
+                case ExitReason.TaskCancelled:
                 default:
                     return null;
             }
@@ -98,8 +100,11 @@ public enum ExitReason
         BadLogSet,
         CompletedSuccessfully,
         IncorrectConfiguration,
+        LogLineTooLong,
         LogSetDoesNotContainRelevantLogs,
         OutOfMemory,
-        UnclassifiedError
+        TaskCancelled,
+        UnclassifiedError,
+        MultipleExitReasonsOnDifferentThreads
     }
 }

LogShark/Dockerfile

@@ -1,7 +1,7 @@
 FROM mcr.microsoft.com/dotnet/core/sdk:3.1-alpine AS build
 WORKDIR /app
 COPY . ./
-RUN dotnet publish "LogShark/LogShark.csproj" -r linux-x64 --self-contained false -c Release /p:Version=4.2.1 -o out
+RUN dotnet publish "LogShark/LogShark.csproj" -r linux-x64 --self-contained false -c Release /p:Version=4.2.2 -o out
 
 FROM mcr.microsoft.com/dotnet/core/runtime:3.1
 WORKDIR /app

LogShark/Extensions/RunSummaryExtensions.cs

@@ -226,6 +226,16 @@ private static void GeneratePublishingResultsSection(StringBuilder sb, RunSummar
             {
                 sb.AppendLine($"Successfully published workbooks: {GenerateHorizontalList(successfullyPublishedWorkbooks)}");
             }
+
+            var skippedWorkbooks = publisherResults.PublishedWorkbooksInfo?
+                .Where(result => result.PublishState == WorkbookPublishResult.WorkbookPublishState.SkippedEmpty)
+                .Select(result => result.OriginalWorkbookName)
+                .OrderBy(workbookName => workbookName)
+                .ToList();
+            if (skippedWorkbooks?.Count > 0)
+            {
+                sb.AppendLine($"Skipped publishing empty workbooks: {GenerateHorizontalList(skippedWorkbooks)}");
+            }
         }
 
         private static void GenerateWorkbookGeneratorResultsSection(StringBuilder sb, RunSummary runSummary)