Skip to content

Commit d464089

Browse files
adinauergetsentry-bot
adinauer
and
getsentry-bot
authored
Report discarded log bytes (#4871)
* Fix log count in client reports * add assertion to test * Format code * report discarded bytes * changelog * log serialization failure * fix javadoc --------- Co-authored-by: Sentry Github Bot <bot+github-bot@sentry.io>
1 parent 591b401 commit d464089

File tree

9 files changed

+180
-3
lines changed

9 files changed

+180
-3
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
- Fallback to distinct-id as user.id logging attribute when user is not set ([#4847](https://github.com/getsentry/sentry-java/pull/4847))
1919
- Report Timber.tag() as `timber.tag` log attribute ([#4845](https://github.com/getsentry/sentry-java/pull/4845))
2020
- Session Replay: Add screenshot strategy serialization to RRWeb events ([#4851](https://github.com/getsentry/sentry-java/pull/4851))
21+
- Report discarded log bytes ([#4871](https://github.com/getsentry/sentry-java/pull/4871))
2122
- Log why a properties file was not loaded ([#4879](https://github.com/getsentry/sentry-java/pull/4879))
2223

2324
### Dependencies

‎sentry/api/sentry.api‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,7 @@ public final class io/sentry/DataCategory : java/lang/Enum {
347347
public static final field Default Lio/sentry/DataCategory;
348348
public static final field Error Lio/sentry/DataCategory;
349349
public static final field Feedback Lio/sentry/DataCategory;
350+
public static final field LogByte Lio/sentry/DataCategory;
350351
public static final field LogItem Lio/sentry/DataCategory;
351352
public static final field Monitor Lio/sentry/DataCategory;
352353
public static final field Profile Lio/sentry/DataCategory;
@@ -7074,6 +7075,7 @@ public final class io/sentry/util/IntegrationUtils {
70747075
public final class io/sentry/util/JsonSerializationUtils {
70757076
public fun <init> ()V
70767077
public static fun atomicIntegerArrayToList (Ljava/util/concurrent/atomic/AtomicIntegerArray;)Ljava/util/List;
7078+
public static fun byteSizeOf (Lio/sentry/ISerializer;Lio/sentry/ILogger;Lio/sentry/JsonSerializable;)J
70777079
public static fun bytesFrom (Lio/sentry/ISerializer;Lio/sentry/ILogger;Lio/sentry/JsonSerializable;)[B
70787080
public static fun calendarToMap (Ljava/util/Calendar;)Ljava/util/Map;
70797081
}

‎sentry/src/main/java/io/sentry/DataCategory.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ public enum DataCategory {
1010
Session("session"),
1111
Attachment("attachment"),
1212
LogItem("log_item"),
13+
LogByte("log_byte"),
1314
Monitor("monitor"),
1415
Profile("profile"),
1516
ProfileChunkUi("profile_chunk_ui"),

‎sentry/src/main/java/io/sentry/SentryClient.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,13 +1184,21 @@ public void captureLog(@Nullable SentryLogEvent logEvent, @Nullable IScope scope
11841184
}
11851185

11861186
if (logEvent != null) {
1187+
final @NotNull SentryLogEvent tmpLogEvent = logEvent;
11871188
logEvent = executeBeforeSendLog(logEvent);
11881189

11891190
if (logEvent == null) {
11901191
options.getLogger().log(SentryLevel.DEBUG, "Log Event was dropped by beforeSendLog");
11911192
options
11921193
.getClientReportRecorder()
11931194
.recordLostEvent(DiscardReason.BEFORE_SEND, DataCategory.LogItem);
1195+
final @NotNull long logEventNumberOfBytes =
1196+
JsonSerializationUtils.byteSizeOf(
1197+
options.getSerializer(), options.getLogger(), tmpLogEvent);
1198+
options
1199+
.getClientReportRecorder()
1200+
.recordLostEvent(
1201+
DiscardReason.BEFORE_SEND, DataCategory.LogByte, logEventNumberOfBytes);
11941202
return;
11951203
}
11961204

‎sentry/src/main/java/io/sentry/clientreport/ClientReportRecorder.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import io.sentry.SentryEnvelopeItem;
77
import io.sentry.SentryItemType;
88
import io.sentry.SentryLevel;
9+
import io.sentry.SentryLogEvent;
910
import io.sentry.SentryLogEvents;
1011
import io.sentry.SentryOptions;
1112
import io.sentry.protocol.SentrySpan;
@@ -104,9 +105,15 @@ public void recordLostEnvelopeItem(
104105
} else if (itemCategory.equals(DataCategory.LogItem)) {
105106
final @Nullable SentryLogEvents logs = envelopeItem.getLogs(options.getSerializer());
106107
if (logs != null) {
107-
final long count = logs.getItems().size();
108+
final @NotNull List<SentryLogEvent> items = logs.getItems();
109+
final long count = items.size();
108110
recordLostEventInternal(reason.getReason(), itemCategory.getCategory(), count);
111+
final long logBytes = envelopeItem.getData().length;
112+
recordLostEventInternal(
113+
reason.getReason(), DataCategory.LogByte.getCategory(), logBytes);
109114
executeOnDiscard(reason, itemCategory, count);
115+
} else {
116+
options.getLogger().log(SentryLevel.ERROR, "Unable to parse lost logs envelope item.");
110117
}
111118
} else {
112119
recordLostEventInternal(reason.getReason(), itemCategory.getCategory(), 1L);

‎sentry/src/main/java/io/sentry/util/JsonSerializationUtils.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,90 @@ public final class JsonSerializationUtils {
6565
return null;
6666
}
6767
}
68+
69+
/**
70+
* Calculates the size in bytes of a serializable object when serialized to JSON without actually
71+
* storing the serialized data. This is more memory efficient than {@link #bytesFrom(ISerializer,
72+
* ILogger, JsonSerializable)} when you only need the size.
73+
*
74+
* @param serializer the serializer
75+
* @param logger the logger
76+
* @param serializable the serializable object
77+
* @return the size in bytes, or 0 if serialization fails
78+
*/
79+
public static long byteSizeOf(
80+
final @NotNull ISerializer serializer,
81+
final @NotNull ILogger logger,
82+
final @Nullable JsonSerializable serializable) {
83+
if (serializable == null) {
84+
return 0;
85+
}
86+
try {
87+
final ByteCountingWriter writer = new ByteCountingWriter();
88+
serializer.serialize(serializable, writer);
89+
return writer.getByteCount();
90+
} catch (Throwable t) {
91+
logger.log(SentryLevel.ERROR, "Could not calculate size of serializable", t);
92+
return 0;
93+
}
94+
}
95+
96+
/**
97+
* A Writer that counts the number of bytes that would be written in UTF-8 encoding without
98+
* actually storing the data.
99+
*/
100+
private static final class ByteCountingWriter extends Writer {
101+
private long byteCount = 0L;
102+
103+
@Override
104+
public void write(final char[] cbuf, final int off, final int len) {
105+
for (int i = off; i < off + len; i++) {
106+
byteCount += utf8ByteCount(cbuf[i]);
107+
}
108+
}
109+
110+
@Override
111+
public void write(final int c) {
112+
byteCount += utf8ByteCount((char) c);
113+
}
114+
115+
@Override
116+
public void write(final @NotNull String str, final int off, final int len) {
117+
for (int i = off; i < off + len; i++) {
118+
byteCount += utf8ByteCount(str.charAt(i));
119+
}
120+
}
121+
122+
@Override
123+
public void flush() {
124+
// Nothing to flush since we don't store data
125+
}
126+
127+
@Override
128+
public void close() {
129+
// Nothing to close
130+
}
131+
132+
public long getByteCount() {
133+
return byteCount;
134+
}
135+
136+
/**
137+
* Calculates the number of bytes needed to encode a character in UTF-8.
138+
*
139+
* @param c the character
140+
* @return the number of bytes (1-4)
141+
*/
142+
private static int utf8ByteCount(final char c) {
143+
if (c <= 0x7F) {
144+
return 1; // ASCII
145+
} else if (c <= 0x7FF) {
146+
return 2; // 2-byte character
147+
} else if (Character.isSurrogate(c)) {
148+
return 2; // Surrogate pair, counted as 2 bytes each (total 4 for the pair)
149+
} else {
150+
return 3; // 3-byte character
151+
}
152+
}
153+
}
68154
}

‎sentry/src/test/java/io/sentry/SentryClientTest.kt‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,10 @@ class SentryClientTest {
293293

294294
assertClientReport(
295295
fixture.sentryOptions.clientReportRecorder,
296-
listOf(DiscardedEvent(DiscardReason.BEFORE_SEND.reason, DataCategory.LogItem.category, 1)),
296+
listOf(
297+
DiscardedEvent(DiscardReason.BEFORE_SEND.reason, DataCategory.LogItem.category, 1),
298+
DiscardedEvent(DiscardReason.BEFORE_SEND.reason, DataCategory.LogByte.category, 109),
299+
),
297300
)
298301
}
299302

@@ -312,7 +315,10 @@ class SentryClientTest {
312315

313316
assertClientReport(
314317
fixture.sentryOptions.clientReportRecorder,
315-
listOf(DiscardedEvent(DiscardReason.BEFORE_SEND.reason, DataCategory.LogItem.category, 1)),
318+
listOf(
319+
DiscardedEvent(DiscardReason.BEFORE_SEND.reason, DataCategory.LogItem.category, 1),
320+
DiscardedEvent(DiscardReason.BEFORE_SEND.reason, DataCategory.LogByte.category, 109),
321+
),
316322
)
317323
}
318324

‎sentry/src/test/java/io/sentry/clientreport/ClientReportTest.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,9 @@ class ClientReportTest {
377377
val logItem =
378378
clientReport!!.discardedEvents!!.first { it.category == DataCategory.LogItem.category }
379379
assertEquals(2, logItem.quantity)
380+
val logByte =
381+
clientReport!!.discardedEvents!!.first { it.category == DataCategory.LogByte.category }
382+
assertEquals(226, logByte.quantity)
380383
}
381384

382385
private fun givenClientReportRecorder(

‎sentry/src/test/java/io/sentry/util/JsonSerializationUtilsTest.kt‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,28 @@ package io.sentry.util
33
import io.sentry.ILogger
44
import io.sentry.JsonSerializable
55
import io.sentry.JsonSerializer
6+
import io.sentry.ObjectWriter
7+
import io.sentry.SentryLogEvent
8+
import io.sentry.SentryLogLevel
9+
import io.sentry.SentryOptions
10+
import io.sentry.protocol.SentryId
611
import java.io.Writer
712
import java.util.Calendar
813
import java.util.concurrent.atomic.AtomicIntegerArray
914
import kotlin.test.Test
1015
import kotlin.test.assertContentEquals
1116
import kotlin.test.assertEquals
1217
import kotlin.test.assertNull
18+
import kotlin.test.assertTrue
1319
import org.mockito.invocation.InvocationOnMock
1420
import org.mockito.kotlin.any
1521
import org.mockito.kotlin.mock
1622

1723
class JsonSerializationUtilsTest {
24+
25+
private val serializer = JsonSerializer(SentryOptions())
26+
private val logger: ILogger = mock()
27+
1828
@Test
1929
fun `serializes calendar to map`() {
2030
val calendar = Calendar.getInstance()
@@ -74,4 +84,57 @@ class JsonSerializationUtilsTest {
7484

7585
assertNull(actualBytes, "Mocker error should be captured and null returned.")
7686
}
87+
88+
@Test
89+
fun `byteSizeOf returns same size as bytesFrom for ASCII`() {
90+
val logEvent = SentryLogEvent(SentryId(), 1234567890.0, "Hello ASCII", SentryLogLevel.INFO)
91+
92+
val actualBytes = JsonSerializationUtils.bytesFrom(serializer, logger, logEvent)
93+
val byteSize = JsonSerializationUtils.byteSizeOf(serializer, logger, logEvent)
94+
95+
assertEquals(
96+
(actualBytes?.size ?: -1).toLong(),
97+
byteSize,
98+
"byteSizeOf should match actual byte array length",
99+
)
100+
assertTrue(byteSize > 0, "byteSize should be positive")
101+
}
102+
103+
@Test
104+
fun `byteSizeOf returns same size as bytesFrom for UTF-8 characters`() {
105+
// Mix of 1-byte, 2-byte, 3-byte and 4-byte UTF-8 characters
106+
val logEvent =
107+
SentryLogEvent(SentryId(), 1234567890.0, "Hello 世界 café 🎉 🚀", SentryLogLevel.WARN)
108+
109+
val actualBytes = JsonSerializationUtils.bytesFrom(serializer, logger, logEvent)
110+
val byteSize = JsonSerializationUtils.byteSizeOf(serializer, logger, logEvent)
111+
112+
assertEquals(
113+
(actualBytes?.size ?: -1).toLong(),
114+
byteSize,
115+
"byteSizeOf should match actual byte array length for UTF-8",
116+
)
117+
assertTrue(byteSize > 0, "byteSize should be positive")
118+
}
119+
120+
@Test
121+
fun `byteSizeOf returns 0 on serialization error`() {
122+
val serializable =
123+
object : JsonSerializable {
124+
override fun serialize(writer: ObjectWriter, logger: ILogger) {
125+
throw RuntimeException("Serialization error")
126+
}
127+
}
128+
129+
val byteSize = JsonSerializationUtils.byteSizeOf(serializer, logger, serializable)
130+
131+
assertEquals(0, byteSize, "byteSizeOf should return 0 on error")
132+
}
133+
134+
@Test
135+
fun `byteSizeOf returns 0 on null serializable`() {
136+
val byteSize = JsonSerializationUtils.byteSizeOf(serializer, logger, null)
137+
138+
assertEquals(0, byteSize, "byteSizeOf should return 0 on null serializable")
139+
}
77140
}

0 commit comments

Comments
 (0)