Synchronize JsonLayout (#246)

loki4j · Jul 2, 2024 · f3f3ea4 · f3f3ea4
1 parent b99211c
commit f3f3ea4
Show file tree

Hide file tree

Showing 4 changed files with 107 additions and 12 deletions.
diff --git a/loki-logback-appender/src/main/java/com/github/loki4j/logback/JsonLayout.java b/loki-logback-appender/src/main/java/com/github/loki4j/logback/JsonLayout.java
@@ -39,14 +39,13 @@ public class JsonLayout extends ContextAwareBase implements Layout<ILoggingEvent
 
     private volatile boolean started;
 
-    private JsonEventWriter jsonWriter;
-
     private List<JsonProvider<ILoggingEvent>> providers;
 
     private List<JsonProvider<ILoggingEvent>> customProviders = new ArrayList<>();
 
     @Override
     public String doLayout(ILoggingEvent event) {
+        var jsonWriter = new JsonEventWriter(INIT_WRITER_CAPACITY_BYTES);
         var standard = providers.iterator();
         var custom = customProviders.iterator();
         var firstFieldWritten = false;
@@ -64,8 +63,6 @@ public String doLayout(ILoggingEvent event) {
 
     @Override
     public void start() {
-        jsonWriter = new JsonEventWriter(INIT_WRITER_CAPACITY_BYTES);
-
         timestamp = ensureProvider(timestamp, TimestampJsonProvider::new);
         loggerName = ensureProvider(loggerName, LoggerNameJsonProvider::new);
         logLevel = ensureProvider(logLevel, LogLevelJsonProvider::new);

diff --git a/loki-logback-appender/src/test/java/com/github/loki4j/logback/JsonLayoutTest.java b/loki-logback-appender/src/test/java/com/github/loki4j/logback/JsonLayoutTest.java
@@ -3,6 +3,13 @@
 import static com.github.loki4j.logback.Generators.*;
 import static org.junit.Assert.*;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 import org.junit.Test;
@@ -13,8 +20,10 @@
 import com.github.loki4j.logback.json.LogLevelJsonProvider;
 import com.github.loki4j.logback.json.LoggerNameJsonProvider;
 import com.github.loki4j.logback.json.MessageJsonProvider;
+import com.github.loki4j.logback.json.StackTraceJsonProvider;
 import com.github.loki4j.logback.json.ThreadNameJsonProvider;
 import com.github.loki4j.logback.json.TimestampJsonProvider;
+
 import com.github.loki4j.testkit.dummy.StringPayload;
 
 import ch.qos.logback.classic.Level;
@@ -50,8 +59,6 @@ public void testWorksInAppender() {
             appender.waitAllAppended();
 
             var actual = StringPayload.parse(sender.lastSendData(), encoder.charset);
-            //System.out.println(expected);
-            //System.out.println(actual);
             assertEquals("jsonLayout", expected, actual);
             return null;
         });
@@ -97,10 +104,70 @@ protected void writeExactlyOneField(JsonEventWriter writer, ILoggingEvent event)
         layout.stop();
     }
 
-    private <T extends JsonProvider<?>> T disable(Supplier<T> ctor) {
-        var provider = ctor.get();
+    @Test
+    public void testThreadSafety() throws Exception {
+        class Result {
+            final String expected;
+            final String actual;
+
+            public Result(final String expected, final String actual) {
+                this.expected = expected;
+                this.actual = actual;
+            }
+        }
+
+        final List<Result> results = Collections.synchronizedList(new ArrayList<>());
+
+        final var layout = jsonMsgLayout();
+        try {
+            layout.setLogLevel(disable(LogLevelJsonProvider::new));
+            layout.setLoggerName(disable(LoggerNameJsonProvider::new));
+            layout.setTimestamp(disable(TimestampJsonProvider::new));
+            layout.setStackTrace(disable(StackTraceJsonProvider::new));
+
+            layout.start();
+
+            // ready...
+            final CountDownLatch latch = new CountDownLatch(1);
+            final ExecutorService executorService = Executors.newFixedThreadPool(16);
+            // set...
+            for (final var event : generateEvents(256, 4)) {
+                executorService.submit(() -> {
+                    awaitLatch(latch);
+                    final String formatted = layout.doLayout(event);
+                    final String expected = String.format("{'thread_name':'%s','message':'%s'}", event.getThreadName(), event.getMessage())
+                            .replace("\n", "\\n")
+                            .replace('\'', '"');
+                    results.add(new Result(expected, formatted));
+                });
+            }
+
+            // go!
+            latch.countDown();
+            executorService.shutdown();
+            assertTrue(executorService.awaitTermination(10L, TimeUnit.SECONDS));
+
+            for (final Result result : results) {
+                assertEquals(result.expected, result.actual);
+            }
+        } finally {
+            layout.stop();
+        }
+    }
+
+    private static void awaitLatch(final CountDownLatch latch) {
+        try {
+            latch.await();
+        } catch (final InterruptedException ex) {
+            Thread.currentThread().interrupt();
+            throw new RuntimeException(ex);
+        }
+    }
+
+    private <T extends JsonProvider<?>> T disable(final Supplier<T> ctor) {
+        final var provider = ctor.get();
         provider.setEnabled(false);
         return provider;
     }
-    
+
 }
diff --git a/loki-logback-appender/src/test/java/com/github/loki4j/logback/performance/AppenderTest.java b/loki-logback-appender/src/test/java/com/github/loki4j/logback/performance/AppenderTest.java
@@ -71,12 +71,12 @@ public void multiThreadPerformance() throws Exception {
                     a -> a.waitAllAppended(),
                     a -> a.stop()),
                 Benchmark.of("dummyJsonAppenderWait",
-                    () -> initApp(capacity, jsonEncoder(false, "singleThreadPerformance")),
+                    () -> initApp(capacity, jsonEncoder(false, "multiThreadPerformance")),
                     (a, e) -> a.append(e),
                     a -> a.waitAllAppended(),
                     a -> a.stop()),
                 Benchmark.of("dummyProtobufAppenderWait",
-                    () -> initApp(capacity, protobufEncoder(false, "singleThreadPerformance")),
+                    () -> initApp(capacity, protobufEncoder(false, "multiThreadPerformance")),
                     (a, e) -> a.append(e),
                     a -> a.waitAllAppended(),
                     a -> a.stop())

diff --git a/loki-logback-appender/src/test/java/com/github/loki4j/logback/performance/LayoutsTest.java b/loki-logback-appender/src/test/java/com/github/loki4j/logback/performance/LayoutsTest.java
@@ -27,7 +27,7 @@ private static Layout<ILoggingEvent> initLayout(Layout<ILoggingEvent> l) {
         l.start();
         return l;
     }
-    
+
     @Test
     @Category({PerformanceTests.class})
     public void layoutPerformance() throws Exception {
@@ -56,4 +56,35 @@ public void layoutPerformance() throws Exception {
         statsLayouts.forEach(System.out::println);
     }
 
+    @Test
+    @Category({PerformanceTests.class})
+    public void multiThreadedLayoutPerformance() throws Exception {
+        var statsLayouts = Benchmarker.run(new Benchmarker.Config<ILoggingEvent>() {{
+            this.runs = 100;
+            this.parFactor = 4;
+            this.generator = () -> InfiniteEventIterator.from(generateEvents(10_000, 10)).limited(100_000);
+            this.benchmarks = Arrays.asList(
+                    Benchmark.of(
+                            "patternLayout",
+                            () -> initLayout(new PatternLayout()),
+                            Layout::doLayout,
+                            layout -> {},
+                            Layout::stop),
+                    Benchmark.of(
+                            "jsonLayout",
+                            () -> initLayout(new JsonLayout()),
+                            Layout::doLayout,
+                            layout -> {},
+                            Layout::stop),
+                    Benchmark.of(
+                            "logstashLayout",
+                            () -> initLayout(new LogstashLayout()),
+                            Layout::doLayout,
+                            layout -> {},
+                            Layout::stop)
+            );
+        }});
+        statsLayouts.forEach(System.out::println);
+    }
+
 }