Skip to content

Commit d036082

Browse files
felixbarnyfelixbarny
authored
Serialize MDC under top level (#67)
Makes it easier to set ECS fields via MDC. Previously, you'd have to configure topLevelLabels for each ECS field added via MDC. Now, if you want to have a field nested under labels, you have to explicitly prefix it with `labels.`.
1 parent 1382236 commit d036082

File tree

12 files changed

+27
-81
lines changed

12 files changed

+27
-81
lines changed

‎README.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,16 @@ We recommend using this library to log into a JSON log file and let Filebeat sen
8989
|[`error.message`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getMessage()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getMessage())|
9090
|[`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getStackTrace()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getStackTrace())|
9191
|[`process.thread.name`](https://www.elastic.co/guide/en/ecs/current/ecs-process.html)|[`LogEvent#getThreadName()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getThreadName()) |
92-
|[`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())|
92+
|Each MDC entry is a top-level field <a href="#note1" id="note1ref"><sup>1</sup></a>|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())|
9393
|[`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextStack()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextStack())|
9494

95+
<a id="note1" href="#note1ref"><sup>1</sup></a> It's recommended to use existing [ECS fields](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for MDC values.
96+
97+
If there is no appropriate ECS field,
98+
consider prefixing your fields with `labels.`, as in `labels.foo`, for simple key/value pairs.
99+
For nested structures consider prefixing with `custom.` to make sure you won't get conflicts if ECS later adds the same fields but with a different mapping.
100+
101+
95102
## Getting Started
96103

97104
### Step 1: Configure application logging

‎ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,10 @@
2626

2727
import java.io.PrintWriter;
2828
import java.io.Writer;
29-
import java.util.Arrays;
30-
import java.util.List;
3129
import java.util.Map;
32-
import java.util.Set;
3330

3431
public class EcsJsonSerializer {
3532

36-
public static final List<String> DEFAULT_TOP_LEVEL_LABELS = Arrays.asList("trace.id", "transaction.id", "span.id", "error.id", "service.name");
3733
private static final TimestampSerializer TIMESTAMP_SERIALIZER = new TimestampSerializer();
3834
private static final ThreadLocal<StringBuilder> messageStringBuilder = new ThreadLocal<StringBuilder>();
3935
private static final String NEW_LINE = System.getProperty("line.separator");
@@ -144,14 +140,11 @@ public static void serializeOrigin(StringBuilder builder, String fileName, Strin
144140
builder.append("},");
145141
}
146142

147-
public static void serializeLabels(StringBuilder builder, Map<String, ?> labels, Set<String> topLevelLabels) {
148-
if (!labels.isEmpty()) {
149-
for (Map.Entry<String, ?> entry : labels.entrySet()) {
143+
public static void serializeMDC(StringBuilder builder, Map<String, ?> properties) {
144+
if (!properties.isEmpty()) {
145+
for (Map.Entry<String, ?> entry : properties.entrySet()) {
150146
builder.append('\"');
151147
String key = entry.getKey();
152-
if (!topLevelLabels.contains(key)) {
153-
builder.append("labels.");
154-
}
155148
JsonUtils.quoteAsString(key, builder);
156149
builder.append("\":\"");
157150
JsonUtils.quoteAsString(toNullSafeString(String.valueOf(entry.getValue())), builder);

‎ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ void testSimpleLog() throws Exception {
6363
void testThreadContext() throws Exception {
6464
putMdc("foo", "bar");
6565
debug("test");
66-
assertThat(getLastLogLine().get("labels.foo").textValue()).isEqualTo("bar");
66+
assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
6767
}
6868

6969
@Test
@@ -75,21 +75,15 @@ void testThreadContextStack() throws Exception {
7575
}
7676

7777
@Test
78-
void testTopLevelLabels() throws Exception {
78+
void testMdc() throws Exception {
7979
putMdc("transaction.id", "0af7651916cd43dd8448eb211c80319c");
8080
putMdc("span.id", "foo");
81+
putMdc("foo", "bar");
8182
debug("test");
8283
assertThat(getLastLogLine().get("labels.transaction.id")).isNull();
8384
assertThat(getLastLogLine().get("transaction.id").textValue()).isEqualTo("0af7651916cd43dd8448eb211c80319c");
8485
assertThat(getLastLogLine().get("span.id").textValue()).isEqualTo("foo");
85-
}
86-
87-
@Test
88-
void testCustomTopLevelLabels() throws Exception {
89-
putMdc("top_level", "foo");
90-
debug("test");
91-
assertThat(getLastLogLine().get("labels.top_level")).isNull();
92-
assertThat(getLastLogLine().get("top_level").textValue()).isEqualTo("foo");
86+
assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
9387
}
9488

9589
@Test

‎log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,10 @@
3030
import org.apache.log4j.spi.LoggingEvent;
3131
import org.apache.log4j.spi.ThrowableInformation;
3232

33-
import java.util.HashSet;
34-
import java.util.Set;
35-
3633
public class EcsLayout extends Layout {
3734

3835
private boolean stackTraceAsArray = false;
3936
private String serviceName;
40-
private Set<String> topLevelLabels = new HashSet<String>(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS);
4137
private boolean includeOrigin;
4238
private String eventDataset;
4339

@@ -51,7 +47,7 @@ public String format(LoggingEvent event) {
5147
EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
5248
EcsJsonSerializer.serializeThreadName(builder, event.getThreadName());
5349
EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName());
54-
EcsJsonSerializer.serializeLabels(builder, event.getProperties(), topLevelLabels);
50+
EcsJsonSerializer.serializeMDC(builder, event.getProperties());
5551
EcsJsonSerializer.serializeTag(builder, event.getNDC());
5652
if (includeOrigin) {
5753
LocationInfo locationInformation = event.getLocationInformation();
@@ -102,10 +98,6 @@ public void setStackTraceAsArray(boolean stackTraceAsArray) {
10298
this.stackTraceAsArray = stackTraceAsArray;
10399
}
104100

105-
public void addTopLevelLabel(String topLevelLabel) {
106-
this.topLevelLabels.add(topLevelLabel);
107-
}
108-
109101
public void setEventDataset(String eventDataset) {
110102
this.eventDataset = eventDataset;
111103
}

‎log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ void setUp() {
5252
ecsLayout.setServiceName("test");
5353
ecsLayout.setStackTraceAsArray(true);
5454
ecsLayout.setIncludeOrigin(true);
55-
ecsLayout.addTopLevelLabel("top_level");
5655
ecsLayout.setEventDataset("testdataset.log");
5756
ecsLayout.activateOptions();
5857
appender.setLayout(ecsLayout);

‎log4j2-ecs-layout/README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ Instead of the usual `<PatternLayout/>`, use `<EcsLayout serviceName="my-app"/>`
4444
|-----------------|-------|-------|-----------|
4545
|serviceName |String | |Sets the `service.name` field so you can filter your logs by a particular service |
4646
|eventDataset |String |`${serviceName}.log`|Sets the `event.dataset` field used by the machine learning job of the Logs app to look for anomalies in the log rate. |
47-
|topLevelLabels |String |`trace.id, transaction.id, span.id, error.id, service.name`|Usually, MDC keys are nested under [`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html). You can specify a comma-separated list of properties which should be on the top level. |
4847
|includeMarkers |boolean|`false`|Log [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as [`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html) |
4948
|stackTraceAsArray|boolean|`false`|Serializes the [`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html) as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex [Filebeat configuration](../README.md#when-stacktraceasarray-is-enabled).|
5049
|includeOrigin |boolean|`false`|If `true`, adds the [`log.origin.file.name`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html), [`log.origin.file.line`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) and [`log.origin.function`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) fields. Note that you also have to set `includeLocation="true"` on your loggers and appenders if you are using the async ones. |

‎log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java‎

Lines changed: 6 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,7 @@
5151
import org.apache.logging.log4j.util.TriConsumer;
5252

5353
import java.nio.charset.Charset;
54-
import java.util.ArrayList;
55-
import java.util.Collection;
56-
import java.util.HashSet;
5754
import java.util.List;
58-
import java.util.Set;
5955
import java.util.concurrent.ConcurrentHashMap;
6056
import java.util.concurrent.ConcurrentMap;
6157

@@ -65,13 +61,10 @@ public class EcsLayout extends AbstractStringLayout {
6561
public static final Charset UTF_8 = Charset.forName("UTF-8");
6662
public static final String[] JSON_FORMAT = {"JSON"};
6763

68-
private final TriConsumer<String, Object, StringBuilder> WRITE_KEY_VALUES_INTO = new TriConsumer<String, Object, StringBuilder>() {
64+
private final TriConsumer<String, Object, StringBuilder> WRITE_MDC = new TriConsumer<String, Object, StringBuilder>() {
6965
@Override
7066
public void accept(final String key, final Object value, final StringBuilder stringBuilder) {
7167
stringBuilder.append('\"');
72-
if (!topLevelLabels.contains(key)) {
73-
stringBuilder.append("labels.");
74-
}
7568
JsonUtils.quoteAsString(key, stringBuilder);
7669
stringBuilder.append("\":\"");
7770
JsonUtils.quoteAsString(EcsJsonSerializer.toNullSafeString(String.valueOf(value)), stringBuilder);
@@ -81,7 +74,6 @@ public void accept(final String key, final Object value, final StringBuilder str
8174

8275
private final KeyValuePair[] additionalFields;
8376
private final PatternFormatter[][] fieldValuePatternFormatter;
84-
private final Set<String> topLevelLabels;
8577
private final boolean stackTraceAsArray;
8678
private final String serviceName;
8779
private final String eventDataset;
@@ -90,13 +82,11 @@ public void accept(final String key, final Object value, final StringBuilder str
9082
private final ConcurrentMap<Class<? extends MultiformatMessage>, Boolean> supportsJson = new ConcurrentHashMap<Class<? extends MultiformatMessage>, Boolean>();
9183
private final ObjectMessageJacksonSerializer objectMessageJacksonSerializer = ObjectMessageJacksonSerializer.Resolver.INSTANCE.resolve();
9284

93-
private EcsLayout(Configuration config, String serviceName, String eventDataset, boolean includeMarkers, KeyValuePair[] additionalFields, Collection<String> topLevelLabels, boolean includeOrigin, boolean stackTraceAsArray) {
85+
private EcsLayout(Configuration config, String serviceName, String eventDataset, boolean includeMarkers, KeyValuePair[] additionalFields, boolean includeOrigin, boolean stackTraceAsArray) {
9486
super(config, UTF_8, null, null);
9587
this.serviceName = serviceName;
9688
this.eventDataset = eventDataset;
9789
this.includeMarkers = includeMarkers;
98-
this.topLevelLabels = new HashSet<String>(topLevelLabels);
99-
this.topLevelLabels.addAll(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS);
10090
this.includeOrigin = includeOrigin;
10191
this.stackTraceAsArray = stackTraceAsArray;
10292
this.additionalFields = additionalFields;
@@ -141,7 +131,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr
141131
EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
142132
EcsJsonSerializer.serializeThreadName(builder, event.getThreadName());
143133
EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName());
144-
serializeLabels(event, builder);
134+
serializeAdditionalFieldsAndMDC(event, builder);
145135
serializeTags(event, builder);
146136
if (includeOrigin) {
147137
EcsJsonSerializer.serializeOrigin(builder, event.getSource());
@@ -151,7 +141,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr
151141
return builder;
152142
}
153143

154-
private void serializeLabels(LogEvent event, StringBuilder builder) {
144+
private void serializeAdditionalFieldsAndMDC(LogEvent event, StringBuilder builder) {
155145
final int length = additionalFields.length;
156146
if (!event.getContextData().isEmpty() || length > 0) {
157147
if (length > 0) {
@@ -185,7 +175,7 @@ private void serializeLabels(LogEvent event, StringBuilder builder) {
185175
}
186176
}
187177
}
188-
event.getContextData().forEach(WRITE_KEY_VALUES_INTO, builder);
178+
event.getContextData().forEach(WRITE_MDC, builder);
189179
}
190180
}
191181

@@ -337,8 +327,6 @@ public static class Builder extends AbstractStringLayout.Builder<EcsLayout.Build
337327
private boolean stackTraceAsArray = false;
338328
@PluginElement("AdditionalField")
339329
private KeyValuePair[] additionalFields = new KeyValuePair[]{};
340-
@PluginBuilderAttribute("topLevelLabels")
341-
private String topLevelLabels;
342330
@PluginBuilderAttribute("includeOrigin")
343331
private boolean includeOrigin = false;
344332

@@ -367,15 +355,6 @@ public boolean isIncludeOrigin() {
367355
return includeOrigin;
368356
}
369357

370-
public String getTopLevelLabels() {
371-
return topLevelLabels;
372-
}
373-
374-
public EcsLayout.Builder setTopLevelLabels(final String topLevelLabels) {
375-
this.topLevelLabels = topLevelLabels;
376-
return asBuilder();
377-
}
378-
379358
/**
380359
* Additional fields to set on each log event.
381360
*
@@ -413,13 +392,7 @@ public EcsLayout.Builder setStackTraceAsArray(boolean stackTraceAsArray) {
413392

414393
@Override
415394
public EcsLayout build() {
416-
List<String> topLevelLabelsList = new ArrayList<String>();
417-
if (topLevelLabels != null) {
418-
for (String label : topLevelLabels.split(",")) {
419-
topLevelLabelsList.add(label.trim());
420-
}
421-
}
422-
return new EcsLayout(getConfiguration(), serviceName, EcsJsonSerializer.computeEventDataset(eventDataset, serviceName), includeMarkers, additionalFields, topLevelLabelsList, includeOrigin, stackTraceAsArray);
395+
return new EcsLayout(getConfiguration(), serviceName, EcsJsonSerializer.computeEventDataset(eventDataset, serviceName), includeMarkers, additionalFields, includeOrigin, stackTraceAsArray);
423396
}
424397

425398
public boolean isStackTraceAsArray() {

‎log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,19 +51,17 @@ void tearDown() throws Exception {
5151
}
5252

5353
@Test
54-
void globalLabels() throws Exception {
54+
void testAdditionalFields() throws Exception {
5555
putMdc("trace.id", "foo");
56-
putMdc("top_level", "foo");
57-
putMdc("nested_under_labels", "foo");
56+
putMdc("foo", "bar");
5857
debug("test");
5958
assertThat(getLastLogLine().get("cluster.uuid").textValue()).isEqualTo("9fe9134b-20b0-465e-acf9-8cc09ac9053b");
6059
assertThat(getLastLogLine().get("node.id").textValue()).isEqualTo("foo");
6160
assertThat(getLastLogLine().get("empty")).isNull();
6261
assertThat(getLastLogLine().get("emptyPattern")).isNull();
6362
assertThat(getLastLogLine().get("clazz").textValue()).startsWith(getClass().getPackageName());
6463
assertThat(getLastLogLine().get("404")).isNull();
65-
assertThat(getLastLogLine().get("top_level").textValue()).isEqualTo("foo");
66-
assertThat(getLastLogLine().get("labels.nested_under_labels").textValue()).isEqualTo("foo");
64+
assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
6765
}
6866

6967
@Test

‎log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ void setUp() {
7070
.setIncludeMarkers(true)
7171
.setIncludeOrigin(true)
7272
.setStackTraceAsArray(true)
73-
.setTopLevelLabels("top_level")
7473
.setEventDataset("testdataset.log")
7574
.setAdditionalFields(new KeyValuePair[]{
7675
new KeyValuePair("cluster.uuid", "9fe9134b-20b0-465e-acf9-8cc09ac9053b"),

‎log4j2-ecs-layout/src/test/resources/log4j2-test.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
</Properties>
66
<Appenders>
77
<List name="TestAppender">
8-
<EcsLayout serviceName="test" includeMarkers="true" includeOrigin="true" stackTraceAsArray="true" topLevelLabels="top_level"
8+
<EcsLayout serviceName="test" includeMarkers="true" includeOrigin="true" stackTraceAsArray="true"
99
eventDataset="testdataset.log">
1010
<KeyValuePair key="cluster.uuid" value="9fe9134b-20b0-465e-acf9-8cc09ac9053b"/>
1111
<KeyValuePair key="node.id" value="${node.id}"/>

0 commit comments

Comments
 (0)