Event dataset (#66)

Author: felixbarny

ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java

@@ -81,6 +81,12 @@ public static void serializeServiceName(StringBuilder builder, String serviceNam
         }
     }
 
+    public static void serializeEventDataset(StringBuilder builder, String eventDataset) {
+        if (eventDataset != null) {
+            builder.append("\"event.dataset\":\"").append(eventDataset).append("\",");
+        }
+    }
+
     public static void serializeLogLevel(StringBuilder builder, String level) {
         builder.append("\"log.level\":");
         // add padding so that all levels line up
@@ -260,6 +266,13 @@ public static StringBuilder getMessageStringBuilder() {
         return result;
     }
 
+    public static String computeEventDataset(String eventDataset, String serviceName) {
+        if (eventDataset == null && serviceName != null && !serviceName.isEmpty()) {
+            return serviceName + ".log";
+        }
+        return eventDataset;
+    }
+
     private static class StringBuilderWriter extends Writer {
 
         private final StringBuilder buffer;

ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java

@@ -50,6 +50,7 @@ void testMetadata() throws Exception {
         assertThat(Instant.parse(getLastLogLine().get("@timestamp").textValue())).isCloseTo(Instant.now(), within(1, ChronoUnit.MINUTES));
         assertThat(getLastLogLine().get("log.level").textValue()).isEqualTo("DEBUG");
         assertThat(getLastLogLine().get("log.logger")).isNotNull();
+        assertThat(getLastLogLine().get("event.dataset").textValue()).isEqualTo("testdataset.log");
     }
 
     @Test

log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java

@@ -39,6 +39,7 @@ public class EcsLayout extends Layout {
     private String serviceName;
     private Set<String> topLevelLabels = new HashSet<String>(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS);
     private boolean includeOrigin;
+    private String eventDataset;
 
     @Override
     public String format(LoggingEvent event) {
@@ -47,6 +48,7 @@ public String format(LoggingEvent event) {
         EcsJsonSerializer.serializeLogLevel(builder, event.getLevel().toString());
         EcsJsonSerializer.serializeFormattedMessage(builder, event.getRenderedMessage());
         EcsJsonSerializer.serializeServiceName(builder, serviceName);
+        EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
         EcsJsonSerializer.serializeThreadName(builder, event.getThreadName());
         EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName());
         EcsJsonSerializer.serializeLabels(builder, event.getProperties(), topLevelLabels);
@@ -85,7 +87,7 @@ public boolean ignoresThrowable() {
 
     @Override
     public void activateOptions() {
-
+        eventDataset = EcsJsonSerializer.computeEventDataset(eventDataset, serviceName);
     }
 
     public void setServiceName(String serviceName) {
@@ -99,8 +101,12 @@ public void setIncludeOrigin(boolean includeOrigin) {
     public void setStackTraceAsArray(boolean stackTraceAsArray) {
         this.stackTraceAsArray = stackTraceAsArray;
     }
-    
+
     public void addTopLevelLabel(String topLevelLabel) {
         this.topLevelLabels.add(topLevelLabel);
     }
+
+    public void setEventDataset(String eventDataset) {
+        this.eventDataset = eventDataset;
+    }
 }

log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java

@@ -53,6 +53,8 @@ void setUp() {
         ecsLayout.setStackTraceAsArray(true);
         ecsLayout.setIncludeOrigin(true);
         ecsLayout.addTopLevelLabel("top_level");
+        ecsLayout.setEventDataset("testdataset.log");
+        ecsLayout.activateOptions();
         appender.setLayout(ecsLayout);
     }
 

log4j2-ecs-layout/README.md

@@ -43,6 +43,7 @@ Instead of the usual `<PatternLayout/>`, use `<EcsLayout serviceName="my-app"/>`
 |Parameter name   |Type   |Default|Description|
 |-----------------|-------|-------|-----------|
 |serviceName      |String |       |Sets the `service.name` field so you can filter your logs by a particular service |
+|eventDataset     |String |`${serviceName}.log`|Sets the `event.dataset` field used by the machine learning job of the Logs app to look for anomalies in the log rate. |
 |topLevelLabels   |String |`trace.id, transaction.id, span.id, error.id, service.name`|Usually, MDC keys are nested under [`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html). You can specify a comma-separated list of properties which should be on the top level. |
 |includeMarkers   |boolean|`false`|Log [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as [`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html) |
 |stackTraceAsArray|boolean|`false`|Serializes the [`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html) as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex [Filebeat configuration](../README.md#when-stacktraceasarray-is-enabled).|

log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java

@@ -84,14 +84,16 @@ public void accept(final String key, final Object value, final StringBuilder str
     private final Set<String> topLevelLabels;
     private final boolean stackTraceAsArray;
     private final String serviceName;
+    private final String eventDataset;
     private final boolean includeMarkers;
     private final boolean includeOrigin;
     private final ConcurrentMap<Class<? extends MultiformatMessage>, Boolean> supportsJson = new ConcurrentHashMap<Class<? extends MultiformatMessage>, Boolean>();
     private final ObjectMessageJacksonSerializer objectMessageJacksonSerializer = ObjectMessageJacksonSerializer.Resolver.INSTANCE.resolve();
 
-    private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, Collection<String> topLevelLabels, boolean includeOrigin, boolean stackTraceAsArray) {
+    private EcsLayout(Configuration config, String serviceName, String eventDataset, boolean includeMarkers, KeyValuePair[] additionalFields, Collection<String> topLevelLabels, boolean includeOrigin, boolean stackTraceAsArray) {
         super(config, UTF_8, null, null);
         this.serviceName = serviceName;
+        this.eventDataset = eventDataset;
         this.includeMarkers = includeMarkers;
         this.topLevelLabels = new HashSet<String>(topLevelLabels);
         this.topLevelLabels.addAll(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS);
@@ -136,6 +138,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr
         EcsJsonSerializer.serializeLogLevel(builder, event.getLevel().toString());
         serializeMessage(builder, gcFree, event.getMessage(), event.getThrown());
         EcsJsonSerializer.serializeServiceName(builder, serviceName);
+        EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
         EcsJsonSerializer.serializeThreadName(builder, event.getThreadName());
         EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName());
         serializeLabels(event, builder);
@@ -326,12 +329,14 @@ public static class Builder extends AbstractStringLayout.Builder<EcsLayout.Build
 
         @PluginBuilderAttribute("serviceName")
         private String serviceName;
+        @PluginBuilderAttribute("eventDataset")
+        private String eventDataset;
         @PluginBuilderAttribute("includeMarkers")
         private boolean includeMarkers = false;
         @PluginBuilderAttribute("stackTraceAsArray")
         private boolean stackTraceAsArray = false;
         @PluginElement("AdditionalField")
-        private KeyValuePair[] additionalFields = new KeyValuePair[] {};
+        private KeyValuePair[] additionalFields = new KeyValuePair[]{};
         @PluginBuilderAttribute("topLevelLabels")
         private String topLevelLabels;
         @PluginBuilderAttribute("includeOrigin")
@@ -350,6 +355,10 @@ public String getServiceName() {
             return serviceName;
         }
 
+        public String getEventDataset() {
+            return eventDataset;
+        }
+
         public boolean isIncludeMarkers() {
             return includeMarkers;
         }
@@ -382,6 +391,11 @@ public EcsLayout.Builder setServiceName(final String serviceName) {
             return asBuilder();
         }
 
+        public EcsLayout.Builder setEventDataset(String eventDataset) {
+            this.eventDataset = eventDataset;
+            return asBuilder();
+        }
+
         public EcsLayout.Builder setIncludeMarkers(final boolean includeMarkers) {
             this.includeMarkers = includeMarkers;
             return asBuilder();
@@ -405,7 +419,7 @@ public EcsLayout build() {
                     topLevelLabelsList.add(label.trim());
                 }
             }
-            return new EcsLayout(getConfiguration(), serviceName, includeMarkers, additionalFields, topLevelLabelsList, includeOrigin, stackTraceAsArray);
+            return new EcsLayout(getConfiguration(), serviceName, EcsJsonSerializer.computeEventDataset(eventDataset, serviceName), includeMarkers, additionalFields, topLevelLabelsList, includeOrigin, stackTraceAsArray);
         }
 
         public boolean isStackTraceAsArray() {

log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java

@@ -71,6 +71,7 @@ void setUp() {
                 .setIncludeOrigin(true)
                 .setStackTraceAsArray(true)
                 .setTopLevelLabels("top_level")
+                .setEventDataset("testdataset.log")
                 .setAdditionalFields(new KeyValuePair[]{
                         new KeyValuePair("cluster.uuid", "9fe9134b-20b0-465e-acf9-8cc09ac9053b"),
                         new KeyValuePair("node.id", "${node.id}"),

log4j2-ecs-layout/src/test/resources/log4j2-test.xml

@@ -5,7 +5,8 @@
     </Properties>
     <Appenders>
         <List name="TestAppender">
-            <EcsLayout serviceName="test" includeMarkers="true" includeOrigin="true" stackTraceAsArray="true" topLevelLabels="top_level">
+            <EcsLayout serviceName="test" includeMarkers="true" includeOrigin="true" stackTraceAsArray="true" topLevelLabels="top_level"
+                       eventDataset="testdataset.log">
                 <KeyValuePair key="cluster.uuid" value="9fe9134b-20b0-465e-acf9-8cc09ac9053b"/>
                 <KeyValuePair key="node.id" value="${node.id}"/>
                 <KeyValuePair key="empty" value="${empty}"/>

logback-ecs-encoder/README.md

@@ -59,6 +59,7 @@ All you have to do is to use the `co.elastic.logging.logback.EcsEncoder` instead
 |Parameter name   |Type   |Default|Description|
 |-----------------|-------|-------|-----------|
 |serviceName      |String |       |Sets the `service.name` field so you can filter your logs by a particular service |
+|eventDataset     |String |`${serviceName}.log`|Sets the `event.dataset` field used by the machine learning job of the Logs app to look for anomalies in the log rate. |
 |includeMarkers   |boolean|`false`|Log [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as [`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html) |
 |stackTraceAsArray|boolean|`false`|Serializes the [`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html) as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex [Filebeat configuration](../README.md#when-stacktraceasarray-is-enabled).|
 |includeOrigin    |boolean|`false`|If `true`, adds the [`log.origin.file.name`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html), [`log.origin.file.line`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) and [`log.origin.function`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) fields. Note that you also have to set `includeLocation="true"` on your loggers and appenders if you are using the async ones. |

logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java

@@ -45,6 +45,7 @@ public class EcsEncoder extends EncoderBase<ILoggingEvent> {
     private static final Charset UTF_8 = Charset.forName("UTF-8");
     private boolean stackTraceAsArray = false;
     private String serviceName;
+    private String eventDataset;
     private boolean includeMarkers = false;
     private ThrowableProxyConverter throwableProxyConverter;
     private Set<String> topLevelLabels = new HashSet<String>(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS);
@@ -61,6 +62,7 @@ public void start() {
         super.start();
         throwableProxyConverter = new ThrowableProxyConverter();
         throwableProxyConverter.start();
+        eventDataset = EcsJsonSerializer.computeEventDataset(eventDataset, serviceName);
     }
 
     @Override
@@ -71,6 +73,7 @@ public byte[] encode(ILoggingEvent event) {
         EcsJsonSerializer.serializeFormattedMessage(builder, event.getFormattedMessage());
         serializeMarkers(event, builder);
         EcsJsonSerializer.serializeServiceName(builder, serviceName);
+        EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
         EcsJsonSerializer.serializeThreadName(builder, event.getThreadName());
         EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName());
         serializeAdditionalFields(builder);
@@ -155,6 +158,10 @@ public void addAdditionalField(Pair pair) {
         this.additionalFields.add(pair);
     }
 
+    public void setEventDataset(String eventDataset) {
+        this.eventDataset = eventDataset;
+    }
+
     public static class Pair {
         private String key;
         private String value = "";