Merge pull request #161 from tableau/Dec2022

4.2.3 Release. Prep plugin and fixes for backgrounder plugin

.gitignore

@@ -25,4 +25,7 @@ TestLogs/
 
 # code coverage
 coverage.json
-coverage.cobertura.xml
+coverage.cobertura.xml
+
+#Other
+*.bak

LogShark.Shared/LogReading/Containers/ContextMetrics.cs

@@ -69,5 +69,8 @@ private string ClientUsernameAlternative
 
         [JsonProperty(PropertyName = "wb")]
         public string Workbook { get; set; }
+
+        [JsonProperty(PropertyName = "trace-id")]
+        public string TraceId { get; set; }
     }
 }

LogShark.Shared/LogReading/LogTypeDetails.cs

@@ -165,7 +165,12 @@ private static IEnumerable<LogTypeInfo> LoadDetails(IProcessingNotificationsColl
                         TsmV0Log("pgsql", "postgresql", "csv"), // TSMv0 - localhost\tabadminagent_0.20181.18.0510.1418770265691097820228\logs\pgsql\postgresql-Mon.csv
                         TsmLog("pgsql", "postgresql", "csv"), // TSM - node2\pgsql_0.20182.18.0627.22303045353787439845635\logs\postgresql-Wed.csv
                     }),
-
+
+                new LogTypeInfo(
+                    logType: LogType.Prep,
+                    logReaderProvider: (stream, filePath) => new PrepLogReader(stream, filePath, processingNotificationsCollector),
+                    fileLocations: PrepLogReader.PrepLogTypeMap.Keys.Select(x => new Regex(x)).ToList()),
+
                 new LogTypeInfo(
                     logType: LogType.ProtocolServer,
                     logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),

LogShark.Shared/LogReading/Readers/PrepLogReader.cs

@@ -0,0 +1,79 @@
+using LogShark.Shared.LogReading.Containers;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace LogShark.Shared.LogReading.Readers
+{
+    internal class PrepLogReader : ILogReader
+    {
+        private readonly String _filePath;
+        private readonly IProcessingNotificationsCollector _processingNotificationsCollector;
+
+        private static readonly char PathDelimiter = '\\';
+
+        private MultilineJavaLogReader _javaLogReader;
+
+        private NativeJsonLogsReader _jsonLogReader;
+
+        public static readonly Dictionary<String, PrepLogTypes> PrepLogTypeMap = new Dictionary<String, PrepLogTypes>()
+        {
+            {@"/floweditor_node.*", PrepLogTypes.MultilineJava},
+            {@"^floweditor_node.*", PrepLogTypes.MultilineJava},
+            {@"/flowprocessor_node.*", PrepLogTypes.MultilineJava},
+            {@"^flowprocessor_node.*", PrepLogTypes.MultilineJava},
+            {@"/nativeapi_flowprocessor.*.txt", PrepLogTypes.NativeJson},
+            {@"^nativeapi_flowprocessor.*.txt", PrepLogTypes.NativeJson},
+            {@"Logs/app.log", PrepLogTypes.NativeJson},
+            {@"^app.log", PrepLogTypes.NativeJson},
+            {@"Logs/preprestapi.*.log*", PrepLogTypes.NativeJson},
+            {@"^preprestapi.*.log*", PrepLogTypes.NativeJson},
+            {@"Logs/log_.*.txt*", PrepLogTypes.NativeJson},
+            {@"^log_.*.txt*", PrepLogTypes.NativeJson},
+        };
+
+        public PrepLogReader(Stream stream, string filePath, IProcessingNotificationsCollector processingNotificationsCollector)
+        {
+            _filePath = filePath;
+            _processingNotificationsCollector = processingNotificationsCollector;
+
+            _jsonLogReader = new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector);
+            _javaLogReader = new MultilineJavaLogReader(stream);
+        }
+
+        public IEnumerable<ReadLogLineResult> ReadLines()
+        {    
+            String filename = _filePath.Split(PathDelimiter).Last();  // Get the file name
+
+            foreach(string pattern in PrepLogTypeMap.Keys)
+            {
+                Regex r = new Regex(pattern);
+                if(r.Match(filename).Success)
+                {
+                    switch(PrepLogTypeMap[pattern])
+                    {
+                        case PrepLogTypes.MultilineJava:
+                            return _javaLogReader.ReadLines();
+                        case PrepLogTypes.NativeJson:
+                            return _jsonLogReader.ReadLines();
+                    }
+                }
+            }
+
+            _processingNotificationsCollector.ReportError($"Can't map file to a type of log, filename: {0}. " +
+                $"Falling back to multiline java log", _filePath);
+
+            return _jsonLogReader.ReadLines();
+        }
+    }
+
+    internal enum PrepLogTypes
+    {
+        NativeJson,
+        MultilineJava,
+    }
+}

LogShark.Shared/LogType.cs

@@ -16,6 +16,7 @@ public enum LogType
         NetstatLinux,
         NetstatWindows,
         PostgresCsv,
+        Prep,
         ProtocolServer,
         SearchServer,
         Tabadmin,

LogShark.Tests/Plugins/PrepPluginTests.cs

@@ -0,0 +1,56 @@
+using FluentAssertions;
+using LogShark.Plugins.Prep;
+using LogShark.Shared;
+using LogShark.Shared.LogReading.Containers;
+using LogShark.Tests.Plugins.Helpers;
+using Microsoft.Extensions.Logging.Abstractions;
+using System;
+using Xunit;
+
+namespace LogShark.Tests.Plugins
+{
+    public class PrepPluginTests : InvariantCultureTestsBase
+    {
+        private static readonly LogFileInfo TestLogFileInfo = new("foo.log", @"prep/foo.log", "worker0", DateTime.MinValue);
+
+
+        [Fact]
+        public void BadInput()
+        {
+            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
+            var testWriterFactory = new TestWriterFactory();
+            using (var plugin = new PrepPlugin())
+            {
+                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
+
+                var wrongContentFormat = new LogLine(new ReadLogLineResult(123, "An invalid Prep log line"), TestLogFileInfo);
+                var nullContent = new LogLine(new ReadLogLineResult(123, null), TestLogFileInfo);
+
+                plugin.ProcessLogLine(nullContent, LogType.Prep);
+                plugin.ProcessLogLine(wrongContentFormat, LogType.Prep);
+            }
+
+            testWriterFactory.AssertAllWritersAreDisposedAndEmpty(1);
+            processingNotificationsCollector.TotalErrorsReported.Should().Be(2);
+        }
+
+
+        [Fact]
+        public void GoodInput()
+        {
+            var processingNotificationsCollector = new ProcessingNotificationsCollector(10);
+            var testWriterFactory = new TestWriterFactory();
+            using (var plugin = new PrepPlugin())
+            {
+                plugin.Configure(testWriterFactory, null, processingNotificationsCollector, new NullLoggerFactory());
+
+                var goodLog = new LogLine(new ReadLogLineResult(123, "2022-06-22 19:37:30.244 -0500 (,,,,) Curator-PathChildrenCache-19 : ERROR org.apache.curator.framework.recipes.cache.PathChildrenCache - "), TestLogFileInfo);
+
+                plugin.ProcessLogLine(goodLog, LogType.Prep);
+            }
+
+            testWriterFactory.GetOneWriterAndVerifyOthersAreEmptyAndDisposed<PrepEvent>("PrepEvents", 1);
+            processingNotificationsCollector.TotalErrorsReported.Should().Be(0);
+        }
+    }
+}