From 67f6aa81d0acebdfe715f9dba900a2c14aa405d0 Mon Sep 17 00:00:00 2001 From: jcjve Date: Fri, 14 Feb 2025 14:29:20 +0100 Subject: [PATCH 1/3] Include HttpEventCollectorResendMiddleware from PR to properly implement retrying As of today the 'bundled' HttpEventCollectorResendMiddleware only retries when '200 - OK' response is returned from the Splunk server, which is odd to say the least. Simon Hege has created a PR on the (https://github.com/splunk/splunk-library-javalogging/pull/287)[main java splunk logging project], but this project seems to be inactive to the point that no contributor is merging this. I suggest to include this in the Quarkus logger, until https://github.com/quarkiverse/quarkus-logging-splunk/issues/281 is implemented. --- .../logging/splunk/SplunkLogHandler.java | 7 +- .../HttpEventCollectorResendMiddleware.java | 125 ++++++++++++++++ ...ttpEventCollectorResendMiddlewareTest.java | 133 ++++++++++++++++++ 3 files changed, 262 insertions(+), 3 deletions(-) create mode 100644 runtime/src/main/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddleware.java create mode 100644 runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java diff --git a/runtime/src/main/java/io/quarkiverse/logging/splunk/SplunkLogHandler.java b/runtime/src/main/java/io/quarkiverse/logging/splunk/SplunkLogHandler.java index 3b6ca18..54b1939 100644 --- a/runtime/src/main/java/io/quarkiverse/logging/splunk/SplunkLogHandler.java +++ b/runtime/src/main/java/io/quarkiverse/logging/splunk/SplunkLogHandler.java @@ -10,11 +10,12 @@ import java.util.logging.Filter; import java.util.logging.Formatter; +import io.quarkiverse.logging.splunk.middleware.HttpEventCollectorResendMiddleware; import org.jboss.logmanager.ExtHandler; import org.jboss.logmanager.ExtLogRecord; import org.jboss.logmanager.filters.AllFilter; -import com.splunk.logging.HttpEventCollectorResendMiddleware; + import com.splunk.logging.HttpEventCollectorSender; public class SplunkLogHandler extends ExtHandler { @@ -45,7 +46,7 @@ public SplunkLogHandler(HttpEventCollectorSender sender, boolean includeExceptio @Override public void doPublish(ExtLogRecord record) { String formatted = formatMessage(record); - if (formatted.length() == 0) { + if (formatted.isEmpty()) { // nothing to write; don't bother return; } @@ -54,7 +55,7 @@ public void doPublish(ExtLogRecord record) { record.getLevel().toString(), formatted, includeLoggerName ? record.getLoggerName() : null, - includeThreadName ? String.format(Locale.US, "%d", record.getThreadID()) : null, + includeThreadName ? String.format(Locale.US, "%d", record.getLongThreadID()) : null, record.getMdcCopy(), (!includeException || record.getThrown() == null) ? null : record.getThrown().getMessage(), null); diff --git a/runtime/src/main/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddleware.java b/runtime/src/main/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddleware.java new file mode 100644 index 0000000..d9863cd --- /dev/null +++ b/runtime/src/main/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddleware.java @@ -0,0 +1,125 @@ +package io.quarkiverse.logging.splunk.middleware; + +import com.splunk.logging.HttpEventCollectorEventInfo; +import com.splunk.logging.HttpEventCollectorMiddleware; + +import java.util.Arrays; +import java.util.HashSet; + +/** + * @copyright + * + * Copyright 2013-2015 Splunk, Inc. + * Derived from https://github.com/splunk/splunk-library-javalogging/pull/287 + * by Simon Hege. + * + * Licensed under the Apache License, Version 2.0 (the "License"): you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import java.util.List; +import java.util.Set; + +/** + * Splunk http event collector resend middleware. + * + * + * HTTP event collector middleware plug in that implements a simple resend policy. + * When HTTP post reply isn't an application error it tries to resend the data. + * An exponentially growing delay is used to prevent server overflow. + */ +public class HttpEventCollectorResendMiddleware + extends HttpEventCollectorMiddleware.HttpSenderMiddleware { + + /** + * List of HTTP event collector server application error statuses. These statuses + * indicate non-transient problems that cannot be fixed by resending the + * data. + */ + private static final Set HttpEventCollectorApplicationErrors = new HashSet<>(Arrays.asList( + // Forbidden + 403, + // Method Not Allowed + 405, + // Bad Request + 400 + )); + private long retriesOnError = 0; + + /** + * Create a resend middleware component. + * @param retriesOnError is the max retry count. + */ + public HttpEventCollectorResendMiddleware(long retriesOnError) { + this.retriesOnError = retriesOnError; + } + + public void postEvents( + final List events, + HttpEventCollectorMiddleware.IHttpSender sender, + HttpEventCollectorMiddleware.IHttpSenderCallback callback) { + callNext(events, sender, new Callback(events, sender, callback)); + } + + private boolean shouldRetry(int statusCode) { + return statusCode != 200 && !HttpEventCollectorApplicationErrors.contains(statusCode); + } + + private class Callback implements HttpEventCollectorMiddleware.IHttpSenderCallback { + private long retries = 0; + private final List events; + private HttpEventCollectorMiddleware.IHttpSenderCallback prevCallback; + private HttpEventCollectorMiddleware.IHttpSender sender; + private final long RetryDelayCeiling = 60 * 1000; // 1 minute + private long retryDelay = 1000; // start with 1 second + + public Callback( + final List events, + HttpEventCollectorMiddleware.IHttpSender sender, + HttpEventCollectorMiddleware.IHttpSenderCallback prevCallback) { + this.events = events; + this.prevCallback = prevCallback; + this.sender = sender; + } + + @Override + public void completed(int statusCode, final String reply) { + if (shouldRetry(statusCode) && retries < retriesOnError) { + retry(); + } else { + // if non-retryable, resend wouldn't help, delegate to previous callback + prevCallback.completed(statusCode, reply); + } + } + + @Override + public void failed(final Exception ex) { + if (retries < retriesOnError) { + retry(); + } else { + prevCallback.failed(ex); + } + } + + private void retry() { + retries++; + try { + Thread.sleep(retryDelay); + callNext(events, sender, this); + } catch (InterruptedException ie) { + prevCallback.failed(ie); + } + // increase delay exponentially + retryDelay = Math.min(RetryDelayCeiling, retryDelay * 2); + } + } +} \ No newline at end of file diff --git a/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java b/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java new file mode 100644 index 0000000..165aee9 --- /dev/null +++ b/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java @@ -0,0 +1,133 @@ +package io.quarkiverse.logging.splunk.middleware; +/** + * @copyright + * + * Copyright 2013-2015 Splunk, Inc. + * Derived from https://github.com/splunk/splunk-library-javalogging/pull/287 + * by Simon Hege. Modifications include updating the unit testing framework to JUnit 5. + * + * Licensed under the Apache License, Version 2.0 (the "License"): you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import com.splunk.logging.HttpEventCollectorEventInfo; +import com.splunk.logging.HttpEventCollectorMiddleware; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class HttpEventCollectorResendMiddlewareTest { + + @Test + public void testPostEvents_whenSuccesShouldNotRetry() { + // Arrange + HttpEventCollectorResendMiddleware middleware = new HttpEventCollectorResendMiddleware(3); + final AtomicInteger callCount = new AtomicInteger(0); + HttpEventCollectorMiddleware.IHttpSender sender = getSender(callCount, 0); + final List recordedStatusCodes = new ArrayList<>(); + final List recordedReplies = new ArrayList<>(); + final List recordedExceptions = new ArrayList<>(); + HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, recordedExceptions); + + // Act + middleware.postEvents(null, sender, callback); + + + // Assert + assertEquals(1, callCount.get()); + assertEquals(1, recordedStatusCodes.size()); + assertEquals(1, recordedReplies.size()); + assertEquals(0, recordedExceptions.size()); + assertEquals(200, recordedStatusCodes.get(0).intValue()); + assertEquals("Success", recordedReplies.get(0)); + } + + @Test + public void testPostEvents_whenUnavailableThenSuccessShouldRetry() { + // Arrange + HttpEventCollectorResendMiddleware middleware = new HttpEventCollectorResendMiddleware(3); + final AtomicInteger callCount = new AtomicInteger(0); + HttpEventCollectorMiddleware.IHttpSender sender = getSender(callCount, 2); + final List recordedStatusCodes = new ArrayList<>(); + final List recordedReplies = new ArrayList<>(); + final List recordedExceptions = new ArrayList<>(); + HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, recordedExceptions); + + // Act + middleware.postEvents(null, sender, callback); + + + // Assert + assertEquals(3, callCount.get()); + assertEquals(1, recordedStatusCodes.size()); + assertEquals(1, recordedReplies.size()); + assertEquals(0, recordedExceptions.size()); + assertEquals(200, recordedStatusCodes.get(0).intValue()); + } + + @Test + public void testPostEvents_whenUnavailableShouldRetryThenStop() { + // Arrange + HttpEventCollectorResendMiddleware middleware = new HttpEventCollectorResendMiddleware(3); + final AtomicInteger callCount = new AtomicInteger(0); + HttpEventCollectorMiddleware.IHttpSender sender = getSender(callCount, 10); + final List recordedStatusCodes = new ArrayList<>(); + final List recordedReplies = new ArrayList<>(); + final List recordedExceptions = new ArrayList<>(); + HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, recordedExceptions); + + // Act + middleware.postEvents(null, sender, callback); + + + // Assert + assertEquals(4, callCount.get()); + assertEquals(1, recordedStatusCodes.size()); + assertEquals(1, recordedReplies.size()); + assertEquals(0, recordedExceptions.size()); + assertEquals(503, recordedStatusCodes.get(0).intValue()); + } + + private static HttpEventCollectorMiddleware.IHttpSender getSender(AtomicInteger callCount, int errorCount) { + return new HttpEventCollectorMiddleware.IHttpSender() { + @Override + public void postEvents(List events, HttpEventCollectorMiddleware.IHttpSenderCallback callback) { + callCount.incrementAndGet(); + if (callCount.get() > errorCount) { + callback.completed(200, "Success"); + } else { + callback.completed(503, "Service Unavailable"); + } + } + }; + } + + private static HttpEventCollectorMiddleware.IHttpSenderCallback getCallback(List recordedStatusCodes, List recordedReplies, List recordedExceptions) { + return new HttpEventCollectorMiddleware.IHttpSenderCallback() { + + @Override + public void completed(int statusCode, String reply) { + recordedStatusCodes.add(statusCode); + recordedReplies.add(reply); + } + + @Override + public void failed(Exception ex) { + recordedExceptions.add(ex); + } + }; + } +} \ No newline at end of file From e9313c8e467c2c59b96a5da8d9bb13760332bb32 Mon Sep 17 00:00:00 2001 From: jcjveraa <3942301+jcjveraa@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:10:12 +0100 Subject: [PATCH 2/3] fix SplunkLogHandlerTest reference to ResenderMiddleware --- .../logging/splunk/SplunkLogHandlerTest.java | 3 +- ...ttpEventCollectorResendMiddlewareTest.java | 61 ++++++------------- 2 files changed, 22 insertions(+), 42 deletions(-) diff --git a/runtime/src/test/java/io/quarkiverse/logging/splunk/SplunkLogHandlerTest.java b/runtime/src/test/java/io/quarkiverse/logging/splunk/SplunkLogHandlerTest.java index 90c3810..dd38c54 100644 --- a/runtime/src/test/java/io/quarkiverse/logging/splunk/SplunkLogHandlerTest.java +++ b/runtime/src/test/java/io/quarkiverse/logging/splunk/SplunkLogHandlerTest.java @@ -23,9 +23,10 @@ import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; -import com.splunk.logging.HttpEventCollectorResendMiddleware; import com.splunk.logging.HttpEventCollectorSender; +import io.quarkiverse.logging.splunk.middleware.HttpEventCollectorResendMiddleware; + @ExtendWith(MockitoExtension.class) class SplunkLogHandlerTest { diff --git a/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java b/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java index 165aee9..cb394fd 100644 --- a/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java +++ b/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java @@ -1,33 +1,14 @@ package io.quarkiverse.logging.splunk.middleware; -/** - * @copyright - * - * Copyright 2013-2015 Splunk, Inc. - * Derived from https://github.com/splunk/splunk-library-javalogging/pull/287 - * by Simon Hege. Modifications include updating the unit testing framework to JUnit 5. - * - * Licensed under the Apache License, Version 2.0 (the "License"): you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -import com.splunk.logging.HttpEventCollectorEventInfo; -import com.splunk.logging.HttpEventCollectorMiddleware; -import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + +import com.splunk.logging.HttpEventCollectorMiddleware; public class HttpEventCollectorResendMiddlewareTest { @@ -40,12 +21,12 @@ public void testPostEvents_whenSuccesShouldNotRetry() { final List recordedStatusCodes = new ArrayList<>(); final List recordedReplies = new ArrayList<>(); final List recordedExceptions = new ArrayList<>(); - HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, recordedExceptions); + HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, + recordedExceptions); // Act middleware.postEvents(null, sender, callback); - // Assert assertEquals(1, callCount.get()); assertEquals(1, recordedStatusCodes.size()); @@ -64,12 +45,12 @@ public void testPostEvents_whenUnavailableThenSuccessShouldRetry() { final List recordedStatusCodes = new ArrayList<>(); final List recordedReplies = new ArrayList<>(); final List recordedExceptions = new ArrayList<>(); - HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, recordedExceptions); + HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, + recordedExceptions); // Act middleware.postEvents(null, sender, callback); - // Assert assertEquals(3, callCount.get()); assertEquals(1, recordedStatusCodes.size()); @@ -87,12 +68,12 @@ public void testPostEvents_whenUnavailableShouldRetryThenStop() { final List recordedStatusCodes = new ArrayList<>(); final List recordedReplies = new ArrayList<>(); final List recordedExceptions = new ArrayList<>(); - HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, recordedExceptions); + HttpEventCollectorMiddleware.IHttpSenderCallback callback = getCallback(recordedStatusCodes, recordedReplies, + recordedExceptions); // Act middleware.postEvents(null, sender, callback); - // Assert assertEquals(4, callCount.get()); assertEquals(1, recordedStatusCodes.size()); @@ -102,20 +83,18 @@ public void testPostEvents_whenUnavailableShouldRetryThenStop() { } private static HttpEventCollectorMiddleware.IHttpSender getSender(AtomicInteger callCount, int errorCount) { - return new HttpEventCollectorMiddleware.IHttpSender() { - @Override - public void postEvents(List events, HttpEventCollectorMiddleware.IHttpSenderCallback callback) { - callCount.incrementAndGet(); - if (callCount.get() > errorCount) { - callback.completed(200, "Success"); - } else { - callback.completed(503, "Service Unavailable"); - } + return (events, callback) -> { + callCount.incrementAndGet(); + if (callCount.get() > errorCount) { + callback.completed(200, "Success"); + } else { + callback.completed(503, "Service Unavailable"); } }; } - private static HttpEventCollectorMiddleware.IHttpSenderCallback getCallback(List recordedStatusCodes, List recordedReplies, List recordedExceptions) { + private static HttpEventCollectorMiddleware.IHttpSenderCallback getCallback(List recordedStatusCodes, + List recordedReplies, List recordedExceptions) { return new HttpEventCollectorMiddleware.IHttpSenderCallback() { @Override @@ -130,4 +109,4 @@ public void failed(Exception ex) { } }; } -} \ No newline at end of file +} From 0929af7f7830694739780201c0cecb535a195e5e Mon Sep 17 00:00:00 2001 From: jcjveraa <3942301+jcjveraa@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:13:59 +0100 Subject: [PATCH 3/3] restore copyright block in HttpEventCollectorResendMiddlewareTest --- ...ttpEventCollectorResendMiddlewareTest.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java b/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java index cb394fd..ffcdeb4 100644 --- a/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java +++ b/runtime/src/test/java/io/quarkiverse/logging/splunk/middleware/HttpEventCollectorResendMiddlewareTest.java @@ -1,4 +1,23 @@ package io.quarkiverse.logging.splunk.middleware; +/** + * @copyright + * + * Copyright 2013-2015 Splunk, Inc. + * Derived from https://github.com/splunk/splunk-library-javalogging/pull/287 + * by Simon Hege. Modifications include updating the unit testing framework to JUnit 5. + * + * Licensed under the Apache License, Version 2.0 (the "License"): you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ import static org.junit.jupiter.api.Assertions.assertEquals; @@ -6,9 +25,9 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import com.splunk.logging.HttpEventCollectorMiddleware; import org.junit.jupiter.api.Test; -import com.splunk.logging.HttpEventCollectorMiddleware; public class HttpEventCollectorResendMiddlewareTest {