Merge pull request #62 from tkowalcz/56-patterns-in-labels

56 patterns in labels

core/src/main/java/pl/tkowalcz/tjahzi/LabelSerializer.java

@@ -32,19 +32,25 @@ public LabelSerializer appendLabelName(String key) {
         return this;
     }
 
-    public void startAppendingLabelValue() {
+    public LabelSerializer startAppendingLabelValue() {
         lastSizePosition = cursor;
         cursor += Integer.BYTES;
+
+        return this;
     }
 
-    public void appendPartialLabelValue(String value) {
-        cursor += buffer.putStringWithoutLengthAscii(cursor, value);
+    public LabelSerializer appendPartialLabelValue(CharSequence value) {
+        cursor += buffer.putStringWithoutLengthAscii(cursor, value.toString());
+
+        return this;
     }
 
-    public void finishAppendingLabelValue() {
+    public LabelSerializer finishAppendingLabelValue() {
         buffer.putInt(
                 lastSizePosition,
                 cursor - lastSizePosition - Integer.BYTES);
+
+        return this;
     }
 
     public LabelSerializer appendLabel(String key, String value) {

core/src/test/java/pl/tkowalcz/tjahzi/LabelSerializerTest.java

@@ -64,20 +64,19 @@ void shouldAppendMultipartLabelsMixedWithRegular() {
         // When
         serializer.appendLabel("Foo", "Bar");
 
-        serializer.appendLabelName("Aliens");
-        serializer.startAppendingLabelValue();
-        serializer.appendPartialLabelValue("Kang");
-        serializer.appendPartialLabelValue("Kodos");
-        serializer.appendPartialLabelValue("Johnson");
-        serializer.finishAppendingLabelValue();
-
-        serializer.appendLabelName("Omicronians");
-        serializer.startAppendingLabelValue();
-        serializer.appendPartialLabelValue("Lrrr");
-        serializer.appendPartialLabelValue("RULER OF THE PLANET OMICRON PERSEI EIGHT");
-        serializer.finishAppendingLabelValue();
-
-        serializer.appendLabel("Popplers", "Problem");
+        serializer
+                .appendLabelName("Aliens")
+                    .startAppendingLabelValue()
+                        .appendPartialLabelValue("Kang")
+                        .appendPartialLabelValue("Kodos")
+                        .appendPartialLabelValue("Johnson")
+                    .finishAppendingLabelValue()
+                .appendLabelName("Omicronians")
+                    .startAppendingLabelValue()
+                        .appendPartialLabelValue("Lrrr")
+                        .appendPartialLabelValue("RULER OF THE PLANET OMICRON PERSEI EIGHT")
+                    .finishAppendingLabelValue()
+                .appendLabel("Popplers", "Problem");
 
         // Then
         assertThat(serializer.getLabelsCount()).isEqualTo(4);

log4j2-appender/README.md

@@ -30,6 +30,7 @@ Log4j2 appender seemed like a good first.
 1. Include minimal appender configuration:
 
 ```xml
+
 <Loki name="loki-appender">
     <host>${sys:loki.host}</host>
     <port>${sys:loki.port}</port>
@@ -45,6 +46,7 @@ Log4j2 appender seemed like a good first.
 1. Reference the appender from inside one of your logger definitions:
 
 ```xml
+
 <Root level="INFO">
     <AppenderRef ref="Loki"/>
 </Root>
@@ -59,13 +61,15 @@ will configure the appender to call to URL: `http://loki.mydomain.com:3100/loki/
 ### Grafana Cloud configuration
 
 Tjahzi can send logs to Grafana Cloud. It needs two things to be configured:
+
 - Set port number to `443` which switches HTTP client into HTTPS mode.
 - Specify `username` and `password` for HTTP basic authentication that Grafana uses.
 
 Password is your "Grafana.com API Key" and can be generated in "Grafana datasource settings". The host in below example
 is just for illustrative purposes.
 
 ```xml
+
 <Loki name="loki-appender">
     <!-- example host -->
     <host>logs-prod-us-central1.grafana.net</host>
@@ -115,12 +119,27 @@ attribute of configuration so that the appender can be found.
 Contents of the properties
 are [automatically interpolated by Log4j2](https://logging.apache.org/log4j/log4j-2.2/manual/configuration.html#PropertySubstitution)
 . All environment, system etc. variable references will be replaced by their values during initialization of the
-appender. The exception is context/MDC (`${ctx:foo}`) value lookup as it make sense only during runtime.
+appender. The exception to this rule is context/MDC (`${ctx:foo}`) value lookup - it is performed for each message at
+runtime (allocation free).
 
 NOTE: This process could have been executed for every lookup type at runtime (for each log message). This approach was
 deemed too expensive. If you need a mechanism to replace a variable (other than context/MDC) after logging system
 initialization I would love to hear your use case - please file an issue.
 
+## Patterns in Labels
+
+Alternative way of specifying label contents is via pattern attribute:
+
+```xml
+
+<Label name="server" pattern="%C{1.}"/>
+```
+
+This pattern is compatible with
+Log4j [pattern layout](https://logging.apache.org/log4j/2.x/manual/layouts.html#PatternLayout). In fact, we reuse lgo4j
+internal classes for this implementation. It is generally efficient and allocation free as
+per [documentation](https://logging.apache.org/log4j/log4j-2.12.1/manual/garbagefree.html#PatternLayout).
+
 ## Details
 
 Let's go through the example config above and analyze configuration options (**Note: Tags are case-insensitive**).
@@ -154,6 +173,13 @@ when [running Loki in multi-tenant mode](https://grafana.com/docs/loki/latest/op
 Specify additional labels attached to each log line sent via this appender instance. See also note
 about [label naming](https://github.com/tkowalcz/tjahzi/wiki/Label-naming).
 
+You can use value attribute to specify static text. You can use `${}` variable substitution inside that text and Tjahzi
+will resolve variables once at startup. If the varaible is a context/MDC lookup it will be resolved dynamically for each
+log line.
+
+This tag also supports `pattern` attribute where you can use pattern layout expressions that will be resolved at
+runtime.
+
 #### LogLevelLabel (optional)
 
 If defined then log level label of configured name will be added to each line sent to Loki. It will contain Log4j log

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java

@@ -122,7 +122,12 @@ public LokiAppender build() {
             );
         }
 
-        LabelFactory labelFactory = new LabelFactory(logLevelLabel, labels);
+        LabelFactory labelFactory = new LabelFactory(
+                getConfiguration(),
+                logLevelLabel,
+                labels
+        );
+
         LabelsDescriptor labelsDescriptor = labelFactory.convertLabelsDroppingInvalid();
         logLevelLabel = labelsDescriptor.getLogLevelLabel();
 

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/labels/AggregateLabelPrinter.java

@@ -18,7 +18,7 @@ public static LabelPrinter of(List<LabelPrinter> result) {
     }
 
     @Override
-    public void append(LogEvent event, Consumer<String> appendable) {
+    public void append(LogEvent event, Consumer<CharSequence> appendable) {
         for (LabelPrinter node : nodes) {
             node.append(event, appendable);
         }

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/labels/Label.java

@@ -7,37 +7,69 @@
 import org.apache.logging.log4j.core.config.plugins.PluginFactory;
 import org.apache.logging.log4j.core.config.plugins.PluginValue;
 import org.apache.logging.log4j.status.StatusLogger;
-import pl.tkowalcz.tjahzi.log4j2.Property;
 
 import java.util.regex.Pattern;
 
 @Plugin(name = "label", category = Node.CATEGORY, printObject = true)
-public class Label extends Property {
+public class Label {
 
     private static final Logger LOGGER = StatusLogger.getLogger();
 
     private static final Pattern LABEL_NAME_PATTER = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*");
 
-    private Label(String name, String value) {
-        super(name, value);
+    private final String name;
+    private final String value;
+    private final String pattern;
+
+    private Label(String name, String value, String pattern) {
+        this.name = name;
+        this.value = value;
+        this.pattern = pattern;
     }
 
     @PluginFactory
     public static Label createLabel(
             @PluginAttribute("name") String name,
-            @PluginValue("value") String value) {
+            @PluginValue("value") String value,
+            @PluginValue("pattern") String pattern
+    ) {
         if (name == null) {
             LOGGER.error("Property name cannot be null");
         }
 
-        return new Label(name, value);
+        if (pattern == null && value == null) {
+            LOGGER.error("Property must have pattern or value specified");
+        }
+
+        return new Label(name, value, pattern);
     }
 
     public boolean hasValidName() {
         return hasValidName(getName());
     }
 
+    public String getName() {
+        return name;
+    }
+
+    public String getPattern() {
+        return pattern;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
     public static boolean hasValidName(String label) {
         return LABEL_NAME_PATTER.matcher(label).matches();
     }
+
+    @Override
+    public String toString() {
+        return "Label{" +
+                "name='" + name + '\'' +
+                ", pattern='" + pattern + '\'' +
+                ", value='" + value + '\'' +
+                '}';
+    }
 }

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/labels/LabelFactory.java

@@ -1,6 +1,9 @@
 package pl.tkowalcz.tjahzi.log4j2.labels;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+import org.apache.logging.log4j.core.pattern.PatternParser;
 import org.apache.logging.log4j.status.StatusLogger;
 import pl.tkowalcz.tjahzi.github.GitHubDocs;
 
@@ -22,9 +25,13 @@ public class LabelFactory {
     private final String logLevelLabel;
     private final Label[] labels;
 
-    public LabelFactory(String logLevelLabel, Label... labels) {
+    private final PatternParser patternParser;
+
+    public LabelFactory(Configuration configuration, String logLevelLabel, Label... labels) {
         this.logLevelLabel = logLevelLabel;
         this.labels = labels;
+
+        this.patternParser = new PatternParser(configuration, PatternLayout.KEY, null);
     }
 
     public LabelsDescriptor convertLabelsDroppingInvalid() {
@@ -95,11 +102,19 @@ private Map<String, LabelPrinter> convertAndLogViolations() {
                 )
                 .collect(toMap(
                         Label::getName,
-                        LabelPrinterFactory::parse,
+                        this::toLabelOrLog4jPattern,
                         (original, duplicate) -> duplicate)
                 );
     }
 
+    private LabelPrinter toLabelOrLog4jPattern(Label label) {
+        if (label.getPattern() != null) {
+            return Log4jAdapterLabelPrinter.of(patternParser.parse(label.getPattern()));
+        }
+
+        return LabelPrinterFactory.parse(label);
+    }
+
     private static String validateLogLevelLabel(
             String logLevelLabel,
             Map<String, ?> staticLabels,

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/labels/LabelPrinter.java

@@ -7,7 +7,7 @@
 
 public interface LabelPrinter {
 
-    void append(LogEvent event, Consumer<String> appendable);
+    void append(LogEvent event, Consumer<CharSequence> appendable);
 
     default boolean isStatic() {
         return false;

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/labels/Literal.java

@@ -22,7 +22,7 @@ public static Literal of(String string) {
     }
 
     @Override
-    public void append(LogEvent event, Consumer<String> appendable) {
+    public void append(LogEvent event, Consumer<CharSequence> appendable) {
         appendable.accept(contents);
     }
 }

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/labels/Log4jAdapterLabelPrinter.java

@@ -0,0 +1,53 @@
+package pl.tkowalcz.tjahzi.log4j2.labels;
+
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.pattern.LiteralPatternConverter;
+import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
+import org.apache.logging.log4j.core.pattern.PatternFormatter;
+
+import java.util.List;
+import java.util.function.Consumer;
+
+@SuppressWarnings("ForLoopReplaceableByForEach")
+public class Log4jAdapterLabelPrinter implements LabelPrinter {
+
+    private final StringBuilder outputBuffer = new StringBuilder();
+    private final List<PatternFormatter> formatters;
+
+    public Log4jAdapterLabelPrinter(List<PatternFormatter> formatters) {
+        this.formatters = formatters;
+    }
+
+    @Override
+    public void append(LogEvent event, Consumer<CharSequence> appendable) {
+        outputBuffer.setLength(0);
+
+        for (int i = 0; i < formatters.size(); i++) {
+            formatters.get(i).format(event, outputBuffer);
+        }
+
+        appendable.accept(outputBuffer);
+    }
+
+    @Override
+    public boolean isStatic() {
+        for (int i = 0; i < formatters.size(); i++) {
+            LogEventPatternConverter converter = formatters.get(i).getConverter();
+
+            if (!(converter instanceof LiteralPatternConverter)) {
+                return false;
+            }
+
+            if (((LiteralPatternConverter) converter).getLiteral().contains("$")) {
+                return false;
+            }
+
+        }
+
+        return true;
+    }
+
+    public static Log4jAdapterLabelPrinter of(List<PatternFormatter> parse) {
+        return new Log4jAdapterLabelPrinter(parse);
+    }
+}

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/labels/MDCLookup.java

@@ -19,7 +19,7 @@ public static MDCLookup of(String variableName, String defaultValue) {
     }
 
     @Override
-    public void append(LogEvent event, Consumer<String> appendable) {
+    public void append(LogEvent event, Consumer<CharSequence> appendable) {
         Object value = event.getContextData().getValue(variableName);
         if (value != null) {
             appendable.accept(value.toString());