Json message layer: support of writing arbitrary objects (#279)

loki-logback-appender/src/main/java/com/github/loki4j/logback/JsonLayout.java

@@ -37,7 +37,7 @@ public class JsonLayout extends ContextAwareBase implements Layout<ILoggingEvent
     private MessageJsonProvider message;
     private StackTraceJsonProvider stackTrace;
     private MdcJsonProvider mdc;
-    private KeyValuePairsJsonProvider keyValuePairs;
+    private KeyValuePairsJsonProvider kvp;
 
     private volatile boolean started;
 
@@ -72,7 +72,7 @@ public void start() {
         message = ensureProvider(message, MessageJsonProvider::new);
         stackTrace = ensureProvider(stackTrace, StackTraceJsonProvider::new);
         mdc = ensureProvider(mdc, MdcJsonProvider::new);
-        keyValuePairs = ensureProvider(keyValuePairs, KeyValuePairsJsonProvider::new);
+        kvp = ensureProvider(kvp, KeyValuePairsJsonProvider::new);
 
         providers = Arrays.asList(
                 timestamp,
@@ -82,7 +82,7 @@ public void start() {
                 message,
                 stackTrace,
                 mdc,
-                keyValuePairs
+                kvp
         );
 
         for (var provider : providers) {
@@ -149,8 +149,8 @@ public void setMdc(MdcJsonProvider mdc) {
         this.mdc = mdc;
     }
 
-    public void setKeyValuePairs(KeyValuePairsJsonProvider keyValuePairs) {
-        this.keyValuePairs = keyValuePairs;
+    public void setKvp(KeyValuePairsJsonProvider keyValuePairs) {
+        this.kvp = keyValuePairs;
     }
 
     @Override

loki-logback-appender/src/main/java/com/github/loki4j/logback/json/AbstractFieldCollectionJsonProvider.java

@@ -0,0 +1,122 @@
+package com.github.loki4j.logback.json;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+
+/**
+ * An abstract provider that writes a collection of JSON fields
+ *
+ * @param <V> Type of the field value supported by this provider.
+ * @param <E> Type of key-value entry served by this provider.
+ * @param <T> Type of key-value collection served by this provider.
+ */
+public abstract class AbstractFieldCollectionJsonProvider<V, E, T extends Collection<E>> extends AbstractJsonProvider {
+
+    /**
+     * A prefix added to all the keys (field names) in this collection.
+     */
+    private String prefix;
+
+    /**
+     * Whether to omit the prefix and use field names from this collection as they are.
+     */
+    private boolean noPrefix;
+
+    /**
+     * A set of keys to exclude from JSON payload.
+     * Exclude list has a precedence over include list.
+     * If not specified, all keys are included.
+     */
+    private Set<String> excludeKeys = new HashSet<>();
+
+    /**
+     * A set of keys to include into JSON payload.
+     * If not specified, all keys are included.
+     */
+    private Set<String> includeKeys = new HashSet<>();
+
+    @Override
+    public boolean canWrite(ILoggingEvent event) {
+        var entries = extractEntries(event);
+        return entries != null && !entries.isEmpty();
+    }
+
+    @Override
+    public boolean writeTo(JsonEventWriter writer, ILoggingEvent event, boolean startWithSeparator) {
+        var entries = extractEntries(event);
+        var firstFieldWritten = false;
+        for (E entry : entries) {
+            var key = extractKey(entry);
+            var value = extractValue(entry);
+            // skip empty records
+            if (key == null || value == null)
+                continue;
+
+            // check exclude list, if defined
+            if (!excludeKeys.isEmpty() && excludeKeys.contains(key))
+                continue;
+
+            // check include list, if defined
+            if (!includeKeys.isEmpty() && !includeKeys.contains(key))
+                continue;
+
+            if (startWithSeparator || firstFieldWritten)
+                writer.writeFieldSeparator();
+            var name = noPrefix ? key : prefix + key;
+            writeField(writer, name, value);
+            firstFieldWritten = true;
+        }
+        return firstFieldWritten;
+    }
+
+    /**
+     * Extract the collection of entries (i.e. fields) from the logging event.
+     * @param event Current logback event.
+     */
+    protected abstract T extractEntries(ILoggingEvent event);
+
+    /**
+     * Extract the key (i.e. field name) from the collection entry.
+     */
+    protected abstract String extractKey(E entry);
+
+    /**
+     * Extract the value (i.e. field value) from the collection entry.
+     */
+    protected abstract V extractValue(E entry);
+
+    /**
+     * Write a field into JSON event layout.
+     * @param writer JSON writer to use.
+     * @param fieldName Name of the field to write.
+     * @param fieldValue Value of the field to write.
+     */
+    protected abstract void writeField(JsonEventWriter writer, String fieldName, V fieldValue);
+
+    public void addExclude(String key) {
+        excludeKeys.add(key);
+    }
+
+    public void addInclude(String key) {
+        includeKeys.add(key);
+    }
+
+    public String getPrefix() {
+        return prefix;
+    }
+
+    public void setPrefix(String prefix) {
+        this.prefix = prefix;
+    }
+
+    public boolean isNoPrefix() {
+        return noPrefix;
+    }
+
+    public void setNoPrefix(boolean noPrefix) {
+        this.noPrefix = noPrefix;
+    }
+}

loki-logback-appender/src/main/java/com/github/loki4j/logback/json/AbstractFieldJsonProvider.java

@@ -1,22 +1,17 @@
 package com.github.loki4j.logback.json;
 
 import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.core.spi.ContextAwareBase;
 
 /**
- * An abstract provider that writes a certain aspect of a logging event as a JSON field
+ * An abstract provider that writes a single JSON field
  */
-public abstract class AbstractFieldJsonProvider extends ContextAwareBase implements JsonProvider<ILoggingEvent> {
-
-    private boolean enabled = true;
+public abstract class AbstractFieldJsonProvider extends AbstractJsonProvider {
 
     /**
      * A JSON field name to use for this provider.
      */
     private String fieldName;
 
-    private volatile boolean started;
-
     @Override
     public boolean canWrite(ILoggingEvent event) {
         return true;
@@ -37,30 +32,6 @@ public boolean writeTo(JsonEventWriter writer, ILoggingEvent event, boolean star
      */
     protected abstract void writeExactlyOneField(JsonEventWriter writer, ILoggingEvent event);
 
-    @Override
-    public void start() {
-        started = true;
-    }
-
-    @Override
-    public void stop() {
-        started = false;
-    }
-
-    @Override
-    public boolean isStarted() {
-        return started;
-    }
-
-    @Override
-    public boolean isEnabled() {
-        return enabled;
-    }
-
-    public void setEnabled(boolean enabled) {
-        this.enabled = enabled;
-    }
-
     public String getFieldName() {
         return fieldName;
     }

loki-logback-appender/src/main/java/com/github/loki4j/logback/json/AbstractJsonProvider.java

@@ -0,0 +1,38 @@
+package com.github.loki4j.logback.json;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.spi.ContextAwareBase;
+
+/**
+ * An abstract provider that writes a certain aspect of a logging event as a JSON fragment
+ */
+public abstract class AbstractJsonProvider extends ContextAwareBase implements JsonProvider<ILoggingEvent> {
+
+    private boolean enabled = true;
+
+    private volatile boolean started;
+
+    @Override
+    public void start() {
+        started = true;
+    }
+
+    @Override
+    public void stop() {
+        started = false;
+    }
+
+    @Override
+    public boolean isStarted() {
+        return started;
+    }
+
+    @Override
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+}

loki-logback-appender/src/main/java/com/github/loki4j/logback/json/JsonEventWriter.java

@@ -2,6 +2,11 @@
 
 import static com.github.loki4j.pkg.dslplatform.json.RawJsonWriter.*;
 
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
 import com.github.loki4j.pkg.dslplatform.json.NumberConverter;
 import com.github.loki4j.pkg.dslplatform.json.RawJsonWriter;
 
@@ -29,8 +34,53 @@ public void writeFieldSeparator() {
     }
 
     public void writeObjectField(String fieldName, Object value) {
-        serializeFieldName(fieldName);
+        writeFieldName(fieldName);
+        writeObjectValue(value);
+    }
+
+    public void writeCustomField(String fieldName, Consumer<JsonEventWriter> write) {
+        writeFieldName(fieldName);
+        write.accept(this);
+    }
+
+    public void writeStringField(String fieldName, String value) {
+        writeFieldName(fieldName);
+        writeStringValue(value);
+    }
+
+    public void writeNumericField(String fieldName, long value) {
+        writeFieldName(fieldName);
+        writeNumericValue(value);
+    }
+
+    public <T> void writeArrayField(String fieldName, T[] values) {
+        writeArrayField(fieldName, values, (w, o) -> w.writeObjectValue(o));
+    }
+
+    public <T> void writeArrayField(String fieldName, T[] values, BiConsumer<JsonEventWriter, T> write) {
+        writeArrayField(fieldName, Arrays.asList(values), write);
+    }
+
+    public <T> void writeArrayField(String fieldName, Iterable<T> values) {
+        writeArrayField(fieldName, values, (w, o) -> w.writeObjectValue(o));
+    }
+
+    public <T> void writeArrayField(String fieldName, Iterable<T> values, BiConsumer<JsonEventWriter, T> write) {
+        writeFieldName(fieldName);
+        writeIteratorValue(values.iterator(), write);
+    }
+
+    public void writeRawJsonField(String fieldName, String rawJson) {
+        writeFieldName(fieldName);
+        writeObjectValue(new RawJsonString(rawJson));
+    }
 
+    private void writeFieldName(String fieldName) {
+        raw.writeString(fieldName);
+        raw.writeByte(SEMI);
+    }
+
+    private void writeObjectValue(Object value) {
         // Object is a reference type, first check the value for null
         if (value == null) {
             raw.writeNull();
@@ -40,34 +90,54 @@ public void writeObjectField(String fieldName, Object value) {
         if (value instanceof String)
             raw.writeString((String) value);
         else if (value instanceof Integer)
-            NumberConverter.serialize(((Integer) value).longValue(), raw);
+            writeNumericValue(((Integer) value).longValue());
         else if (value instanceof Long)
-            NumberConverter.serialize((long) value, raw);
+            writeNumericValue((long) value);
         else if (value instanceof Boolean)
             raw.writeBoolean((boolean) value);
+        else if (value instanceof Iterator<?>)
+            writeIteratorValue((Iterator<?>) value, (w, o) -> w.writeObjectValue(o));
+        else if (value instanceof Iterable)
+            writeIteratorValue(((Iterable<?>) value).iterator(), (w, o) -> w.writeObjectValue(o));
         else if (value instanceof RawJsonString)
             raw.writeRawAscii(((RawJsonString) value).value);
         else
             raw.writeString(value.toString());
     }
 
-    public void writeStringField(String fieldName, String value) {
-        serializeFieldName(fieldName);
+    private void writeStringValue(String value) {
         // String is a reference type, first check the value for null
-        if (value == null)
+        if (value == null) {
             raw.writeNull();
-        else
+        } else {
             raw.writeString(value);
+        }
     }
 
-    public void writeNumericField(String fieldName, long value) {
-        serializeFieldName(fieldName);
+    private void writeNumericValue(long value) {
         NumberConverter.serialize(value, raw);
     }
 
-    private void serializeFieldName(String fieldName) {
-        raw.writeString(fieldName);
-        raw.writeByte(SEMI);
+    private <T> void writeIteratorValue(Iterator<T> it, BiConsumer<JsonEventWriter, T> write) {
+        writeBeginArray();
+        while (it.hasNext()) {
+            write.accept(this, it.next());
+            if (it.hasNext())
+                writeArraySeparator();
+        }
+        writeEndArray();
+    }
+
+    private void writeBeginArray() {
+        raw.writeByte(ARRAY_START);
+    }
+
+    private void writeEndArray() {
+        raw.writeByte(ARRAY_END);
+    }
+
+    private void writeArraySeparator() {
+        raw.writeByte(COMMA);
     }
 
     public String toString() {