Add support for additional fields for log4j, JUL, and JBoss Log Manager (#116)

Author: felixbarny

docs/tab-widgets/ecs-encoder.asciidoc

@@ -73,20 +73,20 @@ Note that this requires a slightly more complex <<setup-stack-trace-as-array, Fi
 |`false`
 |If `true`, adds the {ecs-ref}/ecs-log.html[`log.origin.file.name`],
  {ecs-ref}/ecs-log.html[`log.origin.file.line`] and {ecs-ref}/ecs-log.html[`log.origin.function`] fields.
- Note that you also have to set `includeLocation="true"` on your loggers and appenders if you are using the async ones.
+ Note that you also have to set `<includeCallerData>true</includeCallerData>` on your appenders if you are using the async ones.
 |===
 
 To include any custom field in the output, use following syntax:
 
 [source,xml]
 ----
 <additionalField>
-    <key>foo</key>
-    <value>bar</value>
+    <key>key1</key>
+    <value>value1</value>
 </additionalField>
 <additionalField>
-    <key>baz</key>
-    <value>qux</value>
+    <key>key2</key>
+    <value>value2</value>
 </additionalField>
 ----
 
@@ -156,8 +156,8 @@ To include any custom field in the output, use following syntax:
 [source,xml]
 ----
   <EcsLayout>
-    <KeyValuePair key="additionalField1" value="constant value"/>
-    <KeyValuePair key="additionalField2" value="$${ctx:key}"/>
+    <KeyValuePair key="key1" value="constant value"/>
+    <KeyValuePair key="key2" value="$${ctx:key}"/>
   </EcsLayout>
 ----
 
@@ -178,15 +178,15 @@ For example:
 <log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
     <appender name="LogToConsole" class="org.apache.log4j.ConsoleAppender">
         <param name="Target" value="System.out"/>
-            <layout class="co.elastic.logging.log4j.EcsLayout">
-                <param name="serviceName" value="my-app"/>
-            </layout>
+        <layout class="co.elastic.logging.log4j.EcsLayout">
+            <param name="serviceName" value="my-app"/>
+        </layout>
     </appender>
     <appender name="LogToFile" class="org.apache.log4j.RollingFileAppender">
         <param name="File" value="logs/app.log"/>
-            <layout class="co.elastic.logging.log4j.EcsLayout">
-                <param name="serviceName" value="my-app"/>
-            </layout>
+        <layout class="co.elastic.logging.log4j.EcsLayout">
+            <param name="serviceName" value="my-app"/>
+        </layout>
     </appender>
     <root>
         <priority value="INFO"/>
@@ -195,6 +195,48 @@ For example:
     </root>
 </log4j:configuration>
 ----
+
+
+**Layout Parameters**
+
+|===
+|Parameter name   |Type   |Default |Description
+
+|`serviceName`
+|String
+|
+|Sets the `service.name` field so you can filter your logs by a particular service
+
+|`eventDataset`
+|String
+|`${serviceName}.log`
+|Sets the `event.dataset` field used by the machine learning job of the Logs app to look for anomalies in the log rate.
+
+|`stackTraceAsArray`
+|boolean
+|`false`
+|Serializes the {ecs-ref}/ecs-error.html[`error.stack_trace`] as a JSON array where each element is in a new line to improve readability.
+Note that this requires a slightly more complex <<setup-stack-trace-as-array, Filebeat configuration>>.
+
+|`includeOrigin`
+|boolean
+|`false`
+|If `true`, adds the {ecs-ref}/ecs-log.html[`log.origin.file.name`],
+{ecs-ref}/ecs-log.html[`log.origin.file.line`] and {ecs-ref}/ecs-log.html[`log.origin.function`] fields.
+Note that you also have to set `<param name="LocationInfo" value="true"/>Info` if you are using `AsyncAppender`.
+|===
+
+To include any custom field in the output, use following syntax:
+
+[source,xml]
+----
+<layout class="co.elastic.logging.log4j.EcsLayout">
+   <param name="additionalField" value="key1=value1"/>
+   <param name="additionalField" value="key2=value2"/>
+</layout>
+----
+
+Custom fields are included in the order they are declared.
 // end::log4j[]
 
 // tag::jul[]
@@ -235,6 +277,13 @@ co.elastic.logging.jul.EcsFormatter.serviceName=my-app
 |If `true`, adds the {ecs-ref}/ecs-log.html[`log.origin.file.name`],
  {ecs-ref}/ecs-log.html[`log.origin.file.line`] and {ecs-ref}/ecs-log.html[`log.origin.function`] fields.
  Note that JUL does not stores line number and `log.origin.file.line` will have '1' value.
+
+|`additionalFields`
+|String
+|
+|Adds additional static fields to all log events.
+ The fields are specified as comma-separated key-value pairs.
+ Example: `co.elastic.logging.jul.EcsFormatter.additionalFields=key1=value1,key2=value2`.
 |===
 // end::jul[]
 
@@ -281,5 +330,12 @@ $WILDFLY_HOME/bin/jboss-cli.sh -c '/subsystem=logging/custom-formatter=ECS:add(m
 |`false`
 |If `true`, adds the {ecs-ref}/ecs-log.html[`log.origin.file.name`],
  {ecs-ref}/ecs-log.html[`log.origin.file.line`] and {ecs-ref}/ecs-log.html[`log.origin.function`] fields.
+
+|`additionalFields`
+|String
+|
+|Adds additional static fields to all log events.
+ The fields are specified as comma-separated key-value pairs.
+ Example: `additionalFields=key1=value1,key2=value2`.
 |===
 // end::jboss[]

ecs-logging-core/src/main/java/co/elastic/logging/AdditionalField.java

@@ -0,0 +1,75 @@
+/*-
+ * #%L
+ * Java ECS logging
+ * %%
+ * Copyright (C) 2019 - 2020 Elastic and contributors
+ * %%
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * #L%
+ */
+package co.elastic.logging;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AdditionalField {
+    private String key;
+    private String value = "";
+
+    public AdditionalField() {
+    }
+
+    public AdditionalField(String key, String value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    public static List<AdditionalField> parse(String additionalFields) {
+        String[] split = additionalFields.split(",");
+        ArrayList<AdditionalField> result = new ArrayList<AdditionalField>(split.length);
+        for (String additionalField : split) {
+            AdditionalField field = of(additionalField);
+            result.add(field);
+        }
+        return result;
+    }
+
+    public static AdditionalField of(String additionalField) {
+        String[] keyValue = additionalField.split("=");
+        if (keyValue.length != 2) {
+            throw new IllegalArgumentException("Could not parse " + additionalField);
+        }
+        return new AdditionalField(keyValue[0].trim(), keyValue[1].trim());
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public void setKey(String key) {
+        this.key = key;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+}

ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java

@@ -26,6 +26,7 @@
 
 import java.io.PrintWriter;
 import java.io.Writer;
+import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
 
@@ -282,6 +283,21 @@ public static String computeEventDataset(String eventDataset, String serviceName
         return eventDataset;
     }
 
+    public static void serializeAdditionalFields(StringBuilder builder, List<AdditionalField> additionalFields) {
+        if (!additionalFields.isEmpty()) {
+            for (int i = 0, size = additionalFields.size(); i < size; i++) {
+                AdditionalField additionalField = additionalFields.get(i);
+                if (additionalField.getKey() != null) {
+                    builder.append('\"');
+                    JsonUtils.quoteAsString(additionalField.getKey(), builder);
+                    builder.append("\":\"");
+                    JsonUtils.quoteAsString(additionalField.getValue(), builder);
+                    builder.append("\",");
+                }
+            }
+        }
+    }
+
     private static class StringBuilderWriter extends Writer {
 
         private final StringBuilder buffer;

ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java

@@ -67,6 +67,14 @@ void testMetadata() throws Exception {
         validateLog(getAndValidateLastLogLine());
     }
 
+    @Test
+    final void testAdditionalFields() throws Exception {
+        debug("test");
+        assertThat(getAndValidateLastLogLine().get("key1").textValue()).isEqualTo("value1");
+        assertThat(getAndValidateLastLogLine().get("key2").textValue()).isEqualTo("value2");
+        validateLog(getAndValidateLastLogLine());
+    }
+
     @Test
     void testSimpleLog() throws Exception {
         debug("test");

jboss-logmanager-ecs-formatter/src/main/java/co/elastic/logging/jboss/logmanager/EcsFormatter.java

@@ -25,14 +25,20 @@
 package co.elastic.logging.jboss.logmanager;
 
 import co.elastic.logging.EcsJsonSerializer;
+import co.elastic.logging.AdditionalField;
 import org.jboss.logmanager.ExtFormatter;
 import org.jboss.logmanager.ExtLogRecord;
 import org.jboss.logmanager.LogManager;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
 public class EcsFormatter extends ExtFormatter {
 
     private String serviceName;
     private String eventDataset;
+    private List<AdditionalField> additionalFields = Collections.emptyList();
     private boolean includeOrigin;
     private boolean stackTraceAsArray;
 
@@ -55,6 +61,7 @@ public String format(ExtLogRecord record) {
         EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
         EcsJsonSerializer.serializeThreadName(builder, record.getThreadName());
         EcsJsonSerializer.serializeLoggerName(builder, record.getLoggerName());
+        EcsJsonSerializer.serializeAdditionalFields(builder, additionalFields);
         EcsJsonSerializer.serializeMDC(builder, record.getMdcCopy());
         String ndc = record.getNdc();
         if (ndc != null && !ndc.isEmpty()) {
@@ -92,6 +99,10 @@ public void setEventDataset(String eventDataset) {
         this.eventDataset = eventDataset;
     }
 
+    public void setAdditionalFields(String additionalFields) {
+        this.additionalFields = AdditionalField.parse(additionalFields);
+    }
+
     private String getProperty(final String name, final String defaultValue) {
         String value = LogManager.getLogManager().getProperty(name);
         if (value == null) {

jboss-logmanager-ecs-formatter/src/test/java/co/elastic/logging/jboss/logmanager/JBossLogManagerTest.java

@@ -91,6 +91,7 @@ void setUp() {
         formatter.setStackTraceAsArray(true);
         formatter.setServiceName("test");
         formatter.setEventDataset("testdataset.log");
+        formatter.setAdditionalFields("key1=value1,key2=value2");
 
         logger.setLevel(Level.ALL);
         logger.addHandler(new StreamHandler(byteArrayOutputStream, formatter) {

jul-ecs-formatter/src/main/java/co/elastic/logging/jul/EcsFormatter.java

@@ -24,10 +24,13 @@
  */
 package co.elastic.logging.jul;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.logging.Formatter;
 import java.util.logging.LogManager;
 import java.util.logging.LogRecord;
 
+import co.elastic.logging.AdditionalField;
 import co.elastic.logging.EcsJsonSerializer;
 
 public class EcsFormatter extends Formatter {
@@ -39,6 +42,7 @@ public class EcsFormatter extends Formatter {
     private String serviceName;
     private boolean includeOrigin;
     private String eventDataset;
+    private List<AdditionalField> additionalFields = Collections.emptyList();
 
     /**
      * Default constructor. Will read configuration from LogManager properties.
@@ -59,6 +63,7 @@ public String format(final LogRecord record) {
         EcsJsonSerializer.serializeLogLevel(builder, record.getLevel().getName());
         EcsJsonSerializer.serializeFormattedMessage(builder, super.formatMessage(record));
         EcsJsonSerializer.serializeEcsVersion(builder);
+        EcsJsonSerializer.serializeAdditionalFields(builder, additionalFields);
         EcsJsonSerializer.serializeMDC(builder, mdcSupplier.getMDC());
         EcsJsonSerializer.serializeServiceName(builder, serviceName);
         EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
@@ -95,6 +100,10 @@ public void setEventDataset(String eventDataset) {
         this.eventDataset = eventDataset;
     }
 
+    public void setAdditionalFields(String additionalFields) {
+        this.additionalFields = AdditionalField.parse(additionalFields);
+    }
+
     private String getProperty(final String name, final String defaultValue) {
         String value = LogManager.getLogManager().getProperty(name);
         if (value == null) {
@@ -104,7 +113,7 @@ private String getProperty(final String name, final String defaultValue) {
         }
         return value;
     }
-    
+
     private String buildFileName(String className) {
         String result = UNKNOWN_FILE;
         if (className != null) {
@@ -119,5 +128,4 @@ private String buildFileName(String className) {
         }
         return result;
     }
-
 }

jul-ecs-formatter/src/test/java/co/elastic/logging/jul/EcsFormatterTest.java

@@ -39,7 +39,7 @@ public class EcsFormatterTest {
 
     private final EcsFormatter formatter = new EcsFormatter();
 
-    private final LogRecord record = new LogRecord(Level.INFO, "Example Meesage");
+    private final LogRecord record = new LogRecord(Level.INFO, "Example Message");
     private final ObjectMapper objectMapper = new ObjectMapper();
 
     @BeforeEach

jul-ecs-formatter/src/test/java/co/elastic/logging/jul/JulLoggingTest.java

@@ -116,6 +116,7 @@ void setUp() {
         formatter.setStackTraceAsArray(true);
         formatter.setServiceName("test");
         formatter.setEventDataset("testdataset.log");
+        formatter.setAdditionalFields("key1=value1,key2=value2");
 
         Handler handler = new InMemoryStreamHandler(out, formatter);
         handler.setLevel(Level.ALL);