Add additional ECS fields using nested structure

Update ECS structured logging so that additional fields are written
using a nested structure. A new `Builder` interface has been added to
`StructuredLoggingJsonMembersCustomizer` which can be injected into
formatters and allows `nested` settings to be specified.

Fixes gh-46351

Author: philwebb

spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/ElasticCommonSchemaStructuredLogFormatter.java

@@ -49,8 +49,9 @@
 class ElasticCommonSchemaStructuredLogFormatter extends JsonWriterStructuredLogFormatter<LogEvent> {
 
 	ElasticCommonSchemaStructuredLogFormatter(Environment environment, StackTracePrinter stackTracePrinter,
-			ContextPairs contextPairs, StructuredLoggingJsonMembersCustomizer<?> customizer) {
-		super((members) -> jsonMembers(environment, stackTracePrinter, contextPairs, members), customizer);
+			ContextPairs contextPairs, StructuredLoggingJsonMembersCustomizer.Builder<?> customizerBuilder) {
+		super((members) -> jsonMembers(environment, stackTracePrinter, contextPairs, members),
+				customizerBuilder.nested().build());
 	}
 
 	private static void jsonMembers(Environment environment, StackTracePrinter stackTracePrinter,

spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/StructuredLogLayout.java

@@ -115,10 +115,10 @@ private ElasticCommonSchemaStructuredLogFormatter createEcsFormatter(Instantiato
 			Environment environment = instantiator.getArg(Environment.class);
 			StackTracePrinter stackTracePrinter = instantiator.getArg(StackTracePrinter.class);
 			ContextPairs contextPairs = instantiator.getArg(ContextPairs.class);
-			StructuredLoggingJsonMembersCustomizer<?> jsonMembersCustomizer = instantiator
-				.getArg(StructuredLoggingJsonMembersCustomizer.class);
+			StructuredLoggingJsonMembersCustomizer.Builder<?> jsonMembersCustomizerBuilder = instantiator
+				.getArg(StructuredLoggingJsonMembersCustomizer.Builder.class);
 			return new ElasticCommonSchemaStructuredLogFormatter(environment, stackTracePrinter, contextPairs,
-					jsonMembersCustomizer);
+					jsonMembersCustomizerBuilder);
 		}
 
 		private GraylogExtendedLogFormatStructuredLogFormatter createGraylogFormatter(Instantiator<?> instantiator) {

spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/ElasticCommonSchemaStructuredLogFormatter.java

@@ -53,9 +53,9 @@ class ElasticCommonSchemaStructuredLogFormatter extends JsonWriterStructuredLogF
 
 	ElasticCommonSchemaStructuredLogFormatter(Environment environment, StackTracePrinter stackTracePrinter,
 			ContextPairs contextPairs, ThrowableProxyConverter throwableProxyConverter,
-			StructuredLoggingJsonMembersCustomizer<?> customizer) {
+			StructuredLoggingJsonMembersCustomizer.Builder<?> customizerBuilder) {
 		super((members) -> jsonMembers(environment, stackTracePrinter, contextPairs, throwableProxyConverter, members),
-				customizer);
+				customizerBuilder.nested().build());
 	}
 
 	private static void jsonMembers(Environment environment, StackTracePrinter stackTracePrinter,

spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/StructuredLogEncoder.java

@@ -93,10 +93,10 @@ private StructuredLogFormatter<ILoggingEvent> createEcsFormatter(Instantiator<?>
 		StackTracePrinter stackTracePrinter = instantiator.getArg(StackTracePrinter.class);
 		ContextPairs contextParis = instantiator.getArg(ContextPairs.class);
 		ThrowableProxyConverter throwableProxyConverter = instantiator.getArg(ThrowableProxyConverter.class);
-		StructuredLoggingJsonMembersCustomizer<?> jsonMembersCustomizer = instantiator
-			.getArg(StructuredLoggingJsonMembersCustomizer.class);
+		StructuredLoggingJsonMembersCustomizer.Builder<?> jsonMembersCustomizerBuilder = instantiator
+			.getArg(StructuredLoggingJsonMembersCustomizer.Builder.class);
 		return new ElasticCommonSchemaStructuredLogFormatter(environment, stackTracePrinter, contextParis,
-				throwableProxyConverter, jsonMembersCustomizer);
+				throwableProxyConverter, jsonMembersCustomizerBuilder);
 	}
 
 	private StructuredLogFormatter<ILoggingEvent> createGraylogFormatter(Instantiator<?> instantiator) {

spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatter.java

@@ -30,6 +30,7 @@
  * <ul>
  * <li>{@link Environment}</li>
  * <li>{@link StructuredLoggingJsonMembersCustomizer}</li>
+ * <li>{@link StructuredLoggingJsonMembersCustomizer.Builder}</li>
  * <li>{@link StackTracePrinter} (may be {@code null})</li>
  * <li>{@link ContextPairs}</li>
  * </ul>

spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatterFactory.java

@@ -29,6 +29,7 @@
 import org.springframework.boot.util.Instantiator;
 import org.springframework.boot.util.Instantiator.AvailableParameters;
 import org.springframework.boot.util.Instantiator.FailureHandler;
+import org.springframework.boot.util.LambdaSafe;
 import org.springframework.core.GenericTypeResolver;
 import org.springframework.core.env.Environment;
 import org.springframework.core.io.support.SpringFactoriesLoader;
@@ -85,7 +86,9 @@ public StructuredLogFormatterFactory(Class<E> logEventType, Environment environm
 		this.instantiator = new Instantiator<>(Object.class, (allAvailableParameters) -> {
 			allAvailableParameters.add(Environment.class, environment);
 			allAvailableParameters.add(StructuredLoggingJsonMembersCustomizer.class,
-					(type) -> getStructuredLoggingJsonMembersCustomizer(properties));
+					new JsonMembersCustomizerBuilder(properties).build());
+			allAvailableParameters.add(StructuredLoggingJsonMembersCustomizer.Builder.class,
+					new JsonMembersCustomizerBuilder(properties));
 			allAvailableParameters.add(StackTracePrinter.class, (type) -> getStackTracePrinter(properties));
 			allAvailableParameters.add(ContextPairs.class, (type) -> getContextPairs(properties));
 			if (availableParameters != null) {
@@ -96,30 +99,6 @@ public StructuredLogFormatterFactory(Class<E> logEventType, Environment environm
 		commonFormatters.accept(this.commonFormatters);
 	}
 
-	StructuredLoggingJsonMembersCustomizer<?> getStructuredLoggingJsonMembersCustomizer(
-			StructuredLoggingJsonProperties properties) {
-		List<StructuredLoggingJsonMembersCustomizer<?>> customizers = new ArrayList<>();
-		if (properties != null) {
-			customizers.add(new StructuredLoggingJsonPropertiesJsonMembersCustomizer(this.instantiator, properties));
-		}
-		customizers.addAll(loadStructuredLoggingJsonMembersCustomizers());
-		return (members) -> invokeCustomizers(customizers, members);
-	}
-
-	@SuppressWarnings({ "unchecked", "rawtypes" })
-	private List<StructuredLoggingJsonMembersCustomizer<?>> loadStructuredLoggingJsonMembersCustomizers() {
-		return (List) this.factoriesLoader.load(StructuredLoggingJsonMembersCustomizer.class,
-				ArgumentResolver.from(this.instantiator::getArg));
-	}
-
-	@SuppressWarnings({ "unchecked", "rawtypes" })
-	private void invokeCustomizers(List<StructuredLoggingJsonMembersCustomizer<?>> customizers,
-			Members<Object> members) {
-		for (StructuredLoggingJsonMembersCustomizer<?> customizer : customizers) {
-			((StructuredLoggingJsonMembersCustomizer) customizer).customize(members);
-		}
-	}
-
 	private StackTracePrinter getStackTracePrinter(StructuredLoggingJsonProperties properties) {
 		return (properties != null && properties.stackTrace() != null) ? properties.stackTrace().createPrinter() : null;
 	}
@@ -218,4 +197,53 @@ public interface CommonFormatterFactory<E> {
 
 	}
 
+	/**
+	 * {@link StructuredLoggingJsonMembersCustomizer.Builder} implementation.
+	 */
+	class JsonMembersCustomizerBuilder implements StructuredLoggingJsonMembersCustomizer.Builder<E> {
+
+		private final StructuredLoggingJsonProperties properties;
+
+		private boolean nested;
+
+		JsonMembersCustomizerBuilder(StructuredLoggingJsonProperties properties) {
+			this.properties = properties;
+		}
+
+		@Override
+		public JsonMembersCustomizerBuilder nested(boolean nested) {
+			this.nested = nested;
+			return this;
+		}
+
+		@Override
+		public StructuredLoggingJsonMembersCustomizer<E> build() {
+			return (members) -> {
+				List<StructuredLoggingJsonMembersCustomizer<?>> customizers = new ArrayList<>();
+				if (this.properties != null) {
+					customizers.add(new StructuredLoggingJsonPropertiesJsonMembersCustomizer(
+							StructuredLogFormatterFactory.this.instantiator, this.properties, this.nested));
+				}
+				customizers.addAll(loadStructuredLoggingJsonMembersCustomizers());
+				invokeCustomizers(members, customizers);
+			};
+		}
+
+		@SuppressWarnings({ "unchecked", "rawtypes" })
+		private List<StructuredLoggingJsonMembersCustomizer<?>> loadStructuredLoggingJsonMembersCustomizers() {
+			return (List) StructuredLogFormatterFactory.this.factoriesLoader.load(
+					StructuredLoggingJsonMembersCustomizer.class,
+					ArgumentResolver.from(StructuredLogFormatterFactory.this.instantiator::getArg));
+		}
+
+		@SuppressWarnings("unchecked")
+		private void invokeCustomizers(Members<E> members,
+				List<StructuredLoggingJsonMembersCustomizer<?>> customizers) {
+			LambdaSafe.callbacks(StructuredLoggingJsonMembersCustomizer.class, customizers, members)
+				.withFilter(LambdaSafe.Filter.allowAll())
+				.invoke((customizer) -> customizer.customize(members));
+		}
+
+	}
+
 }

spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLoggingJsonMembersCustomizer.java

@@ -55,4 +55,37 @@ public interface StructuredLoggingJsonMembersCustomizer<T> {
 	 */
 	void customize(JsonWriter.Members<T> members);
 
+	/**
+	 * Builder that can be injected into a {@link StructuredLogFormatter} to build the
+	 * {@link StructuredLoggingJsonMembersCustomizer} when specific settings are required.
+	 *
+	 * @param <T> the type being written
+	 * @since 3.5.4
+	 */
+	interface Builder<T> {
+
+		/**
+		 * Use nested fields when adding JSON from user defined properties.
+		 * @return this builder
+		 */
+		default Builder<T> nested() {
+			return nested(true);
+		}
+
+		/**
+		 * Set if nested fields should be used when adding JSON from user defined
+		 * properties.
+		 * @param nested if nested fields are to be used
+		 * @return this builder
+		 */
+		Builder<T> nested(boolean nested);
+
+		/**
+		 * Build the {@link StructuredLoggingJsonMembersCustomizer}.
+		 * @return the built customizer
+		 */
+		StructuredLoggingJsonMembersCustomizer<T> build();
+
+	}
+
 }

spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLoggingJsonPropertiesJsonMembersCustomizer.java

@@ -36,10 +36,13 @@ class StructuredLoggingJsonPropertiesJsonMembersCustomizer implements Structured
 
 	private final StructuredLoggingJsonProperties properties;
 
+	private final boolean nested;
+
 	StructuredLoggingJsonPropertiesJsonMembersCustomizer(Instantiator<?> instantiator,
-			StructuredLoggingJsonProperties properties) {
+			StructuredLoggingJsonProperties properties, boolean nested) {
 		this.instantiator = instantiator;
 		this.properties = properties;
+		this.nested = nested;
 	}
 
 	@Override
@@ -48,7 +51,13 @@ public void customize(Members<Object> members) {
 		members.applyingNameProcessor(this::renameJsonMembers);
 		Map<String, String> add = this.properties.add();
 		if (!CollectionUtils.isEmpty(add)) {
-			add.forEach(members::add);
+			if (this.nested) {
+				ContextPairs contextPairs = new ContextPairs(true, "");
+				members.add().usingPairs(contextPairs.nested((pairs) -> pairs.addMapEntries((source) -> add)));
+			}
+			else {
+				add.forEach(members::add);
+			}
 		}
 		this.properties.customizers(this.instantiator).forEach((customizer) -> customizer.customize(members));
 	}

spring-boot-project/spring-boot/src/test/java/org/springframework/boot/logging/log4j2/AbstractStructuredLoggingTests.java

@@ -31,6 +31,7 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
+import org.springframework.boot.logging.structured.MockStructuredLoggingJsonMembersCustomizerBuilder;
 import org.springframework.boot.logging.structured.StructuredLoggingJsonMembersCustomizer;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -50,6 +51,9 @@ abstract class AbstractStructuredLoggingTests {
 	@Mock
 	StructuredLoggingJsonMembersCustomizer<?> customizer;
 
+	MockStructuredLoggingJsonMembersCustomizerBuilder<?> customizerBuilder = new MockStructuredLoggingJsonMembersCustomizerBuilder<>(
+			() -> this.customizer);
+
 	protected Map<String, Object> map(Object... values) {
 		assertThat(values.length).isEven();
 		Map<String, Object> result = new HashMap<>();

spring-boot-project/spring-boot/src/test/java/org/springframework/boot/logging/log4j2/ElasticCommonSchemaStructuredLogFormatterTests.java

@@ -56,7 +56,12 @@ void setUp() {
 		this.environment.setProperty("logging.structured.ecs.service.node-name", "node-1");
 		this.environment.setProperty("spring.application.pid", "1");
 		this.formatter = new ElasticCommonSchemaStructuredLogFormatter(this.environment, null,
-				TestContextPairs.include(), this.customizer);
+				TestContextPairs.include(), this.customizerBuilder);
+	}
+
+	@Test
+	void callsNestedOnCustomizerBuilder() {
+		assertThat(this.customizerBuilder.isNested()).isTrue();
 	}
 
 	@Test
@@ -109,7 +114,7 @@ void shouldFormatException() {
 	@SuppressWarnings("unchecked")
 	void shouldFormatExceptionUsingStackTracePrinter() {
 		this.formatter = new ElasticCommonSchemaStructuredLogFormatter(this.environment, new SimpleStackTracePrinter(),
-				TestContextPairs.include(), this.customizer);
+				TestContextPairs.include(), this.customizerBuilder);
 		MutableLogEvent event = createEvent();
 		event.setThrown(new RuntimeException("Boom"));
 		Map<String, Object> deserialized = deserialize(this.formatter.format(event));