feat: Add attachments API (#5004)

- Adds `Attachment` interface and extends `AttachmentItemHeaders` with required fields
- Adds `addAttachment`, `getAttachments` and `clearAttachments` to `Scope`
- Adds `attachments: Attachment[]` to `EventHint`
- In the private capture/process/prepare/send methods in the `BaseClient`, the `EventHint` argument has moved as it's no longer optional
  - We can't mutate the hint if it's undefined!
- Copies attachments from the scope to the hint in `_prepareEvent`
- Adds optional `textEncoder` to `InternalBaseTransportOptions` and overrides this in the node client to support node 8-10
  - Manually pass `new TextEncoder()` in many of the tests so they pass on node.js 8-10
- Adds binary serialisation support for envelopes
  - `serializeEnvelope` returns `string | Uint8Array` which all transports supported without modification
  - Defaults to concatenating strings when no attachments are found. String concatenation is about 10x faster than the binary serialisation 
- Rewrites `parseEnvelope` in `testutils.ts` so that it can parse binary envelopes

Author: timfish

packages/browser/src/client.ts

@@ -104,7 +104,7 @@ export class BrowserClient extends BaseClient<BrowserClientOptions> {
   /**
    * @inheritDoc
    */
-  public sendEvent(event: Event): void {
+  public sendEvent(event: Event, hint?: EventHint): void {
     // We only want to add the sentry event breadcrumb when the user has the breadcrumb integration installed and
     // activated its `sentry` option.
     // We also do not want to use the `Breadcrumbs` class here directly, because we do not want it to be included in
@@ -133,15 +133,15 @@ export class BrowserClient extends BaseClient<BrowserClientOptions> {
       );
     }
 
-    super.sendEvent(event);
+    super.sendEvent(event, hint);
   }
 
   /**
    * @inheritDoc
    */
-  protected _prepareEvent(event: Event, scope?: Scope, hint?: EventHint): PromiseLike<Event | null> {
+  protected _prepareEvent(event: Event, hint: EventHint, scope?: Scope): PromiseLike<Event | null> {
     event.platform = event.platform || 'javascript';
-    return super._prepareEvent(event, scope, hint);
+    return super._prepareEvent(event, hint, scope);
   }
 
   /**

packages/browser/src/transports/utils.ts

@@ -86,7 +86,7 @@ export function getNativeFetchImplementation(): FetchImpl {
  * @param url report endpoint
  * @param body report payload
  */
-export function sendReport(url: string, body: string): void {
+export function sendReport(url: string, body: string | Uint8Array): void {
   const isRealNavigator = Object.prototype.toString.call(global && global.navigator) === '[object Navigator]';
   const hasSendBeacon = isRealNavigator && typeof global.navigator.sendBeacon === 'function';
 

packages/browser/test/unit/transports/fetch.test.ts

@@ -1,5 +1,6 @@
 import { EventEnvelope, EventItem } from '@sentry/types';
 import { createEnvelope, serializeEnvelope } from '@sentry/utils';
+import { TextEncoder } from 'util';
 
 import { makeFetchTransport } from '../../../src/transports/fetch';
 import { BrowserTransportOptions } from '../../../src/transports/types';
@@ -8,6 +9,7 @@ import { FetchImpl } from '../../../src/transports/utils';
 const DEFAULT_FETCH_TRANSPORT_OPTIONS: BrowserTransportOptions = {
   url: 'https://sentry.io/api/42/store/?sentry_key=123&sentry_version=7',
   recordDroppedEvent: () => undefined,
+  textEncoder: new TextEncoder(),
 };
 
 const ERROR_ENVELOPE = createEnvelope<EventEnvelope>({ event_id: 'aa3ff046696b4bc6b609ce6d28fde9e2', sent_at: '123' }, [
@@ -40,7 +42,7 @@ describe('NewFetchTransport', () => {
     expect(mockFetch).toHaveBeenCalledTimes(1);
 
     expect(mockFetch).toHaveBeenLastCalledWith(DEFAULT_FETCH_TRANSPORT_OPTIONS.url, {
-      body: serializeEnvelope(ERROR_ENVELOPE),
+      body: serializeEnvelope(ERROR_ENVELOPE, new TextEncoder()),
       method: 'POST',
       referrerPolicy: 'origin',
     });
@@ -90,7 +92,7 @@ describe('NewFetchTransport', () => {
 
     await transport.send(ERROR_ENVELOPE);
     expect(mockFetch).toHaveBeenLastCalledWith(DEFAULT_FETCH_TRANSPORT_OPTIONS.url, {
-      body: serializeEnvelope(ERROR_ENVELOPE),
+      body: serializeEnvelope(ERROR_ENVELOPE, new TextEncoder()),
       method: 'POST',
       ...REQUEST_OPTIONS,
     });

packages/browser/test/unit/transports/xhr.test.ts

@@ -1,12 +1,14 @@
 import { EventEnvelope, EventItem } from '@sentry/types';
 import { createEnvelope, serializeEnvelope } from '@sentry/utils';
+import { TextEncoder } from 'util';
 
 import { BrowserTransportOptions } from '../../../src/transports/types';
 import { makeXHRTransport } from '../../../src/transports/xhr';
 
 const DEFAULT_XHR_TRANSPORT_OPTIONS: BrowserTransportOptions = {
   url: 'https://sentry.io/api/42/store/?sentry_key=123&sentry_version=7',
   recordDroppedEvent: () => undefined,
+  textEncoder: new TextEncoder(),
 };
 
 const ERROR_ENVELOPE = createEnvelope<EventEnvelope>({ event_id: 'aa3ff046696b4bc6b609ce6d28fde9e2', sent_at: '123' }, [
@@ -64,7 +66,7 @@ describe('NewXHRTransport', () => {
     expect(xhrMock.open).toHaveBeenCalledTimes(1);
     expect(xhrMock.open).toHaveBeenCalledWith('POST', DEFAULT_XHR_TRANSPORT_OPTIONS.url);
     expect(xhrMock.send).toHaveBeenCalledTimes(1);
-    expect(xhrMock.send).toHaveBeenCalledWith(serializeEnvelope(ERROR_ENVELOPE));
+    expect(xhrMock.send).toHaveBeenCalledWith(serializeEnvelope(ERROR_ENVELOPE, new TextEncoder()));
   });
 
   it('sets rate limit response headers', async () => {

packages/core/src/baseclient.ts

@@ -19,7 +19,9 @@ import {
   Transport,
 } from '@sentry/types';
 import {
+  addItemToEnvelope,
   checkOrSetAlreadyCaught,
+  createAttachmentEnvelopeItem,
   dateTimestampInSeconds,
   isPlainObject,
   isPrimitive,
@@ -283,9 +285,14 @@ export abstract class BaseClient<O extends ClientOptions> implements Client<O> {
   /**
    * @inheritDoc
    */
-  public sendEvent(event: Event): void {
+  public sendEvent(event: Event, hint: EventHint = {}): void {
     if (this._dsn) {
       const env = createEventEnvelope(event, this._dsn, this._options._metadata, this._options.tunnel);
+
+      for (const attachment of hint.attachments || []) {
+        addItemToEnvelope(env, createAttachmentEnvelopeItem(attachment, this._options.transportOptions?.textEncoder));
+      }
+
       this._sendEnvelope(env);
     }
   }
@@ -401,11 +408,11 @@ export abstract class BaseClient<O extends ClientOptions> implements Client<O> {
    * @param scope A scope containing event metadata.
    * @returns A new event with more information.
    */
-  protected _prepareEvent(event: Event, scope?: Scope, hint?: EventHint): PromiseLike<Event | null> {
+  protected _prepareEvent(event: Event, hint: EventHint, scope?: Scope): PromiseLike<Event | null> {
     const { normalizeDepth = 3, normalizeMaxBreadth = 1_000 } = this.getOptions();
     const prepared: Event = {
       ...event,
-      event_id: event.event_id || (hint && hint.event_id ? hint.event_id : uuid4()),
+      event_id: event.event_id || hint.event_id || uuid4(),
       timestamp: event.timestamp || dateTimestampInSeconds(),
     };
 
@@ -415,7 +422,7 @@ export abstract class BaseClient<O extends ClientOptions> implements Client<O> {
     // If we have scope given to us, use it as the base for further modifications.
     // This allows us to prevent unnecessary copying of data if `captureContext` is not provided.
     let finalScope = scope;
-    if (hint && hint.captureContext) {
+    if (hint.captureContext) {
       finalScope = Scope.clone(finalScope).update(hint.captureContext);
     }
 
@@ -425,6 +432,13 @@ export abstract class BaseClient<O extends ClientOptions> implements Client<O> {
     // This should be the last thing called, since we want that
     // {@link Hub.addEventProcessor} gets the finished prepared event.
     if (finalScope) {
+      // Collect attachments from the hint and scope
+      const attachments = [...(hint.attachments || []), ...finalScope.getAttachments()];
+
+      if (attachments.length) {
+        hint.attachments = attachments;
+      }
+
       // In case we have a hub we reassign it.
       result = finalScope.applyToEvent(prepared, hint);
     }
@@ -552,7 +566,7 @@ export abstract class BaseClient<O extends ClientOptions> implements Client<O> {
    * @param hint
    * @param scope
    */
-  protected _captureEvent(event: Event, hint?: EventHint, scope?: Scope): PromiseLike<string | undefined> {
+  protected _captureEvent(event: Event, hint: EventHint = {}, scope?: Scope): PromiseLike<string | undefined> {
     return this._processEvent(event, hint, scope).then(
       finalEvent => {
         return finalEvent.event_id;
@@ -577,7 +591,7 @@ export abstract class BaseClient<O extends ClientOptions> implements Client<O> {
    * @param scope A scope containing event metadata.
    * @returns A SyncPromise that resolves with the event or rejects in case event was/will not be send.
    */
-  protected _processEvent(event: Event, hint?: EventHint, scope?: Scope): PromiseLike<Event> {
+  protected _processEvent(event: Event, hint: EventHint, scope?: Scope): PromiseLike<Event> {
     const { beforeSend, sampleRate } = this.getOptions();
 
     if (!this._isEnabled()) {
@@ -597,14 +611,14 @@ export abstract class BaseClient<O extends ClientOptions> implements Client<O> {
       );
     }
 
-    return this._prepareEvent(event, scope, hint)
+    return this._prepareEvent(event, hint, scope)
       .then(prepared => {
         if (prepared === null) {
           this.recordDroppedEvent('event_processor', event.type || 'error');
           throw new SentryError('An event processor returned null, will not send event.');
         }
 
-        const isInternalException = hint && hint.data && (hint.data as { __sentry__: boolean }).__sentry__ === true;
+        const isInternalException = hint.data && (hint.data as { __sentry__: boolean }).__sentry__ === true;
         if (isInternalException || isTransaction || !beforeSend) {
           return prepared;
         }
@@ -623,7 +637,7 @@ export abstract class BaseClient<O extends ClientOptions> implements Client<O> {
           this._updateSessionFromEvent(session, processedEvent);
         }
 
-        this.sendEvent(processedEvent);
+        this.sendEvent(processedEvent, hint);
         return processedEvent;
       })
       .then(null, reason => {

packages/core/src/transports/base.ts

@@ -69,7 +69,7 @@ export function createTransport(
     };
 
     const requestTask = (): PromiseLike<void> =>
-      makeRequest({ body: serializeEnvelope(filteredEnvelope) }).then(
+      makeRequest({ body: serializeEnvelope(filteredEnvelope, options.textEncoder) }).then(
         response => {
           // We don't want to throw on NOK responses, but we want to at least log them
           if (response.statusCode !== undefined && (response.statusCode < 200 || response.statusCode >= 300)) {

packages/core/test/lib/transports/base.test.ts

@@ -1,5 +1,6 @@
 import { EventEnvelope, EventItem, TransportMakeRequestResponse } from '@sentry/types';
 import { createEnvelope, PromiseBuffer, resolvedSyncPromise, serializeEnvelope } from '@sentry/utils';
+import { TextEncoder } from 'util';
 
 import { createTransport } from '../../../src/transports/base';
 
@@ -14,6 +15,7 @@ const TRANSACTION_ENVELOPE = createEnvelope<EventEnvelope>(
 
 const transportOptions = {
   recordDroppedEvent: () => undefined, // noop
+  textEncoder: new TextEncoder(),
 };
 
 describe('createTransport', () => {
@@ -36,7 +38,7 @@ describe('createTransport', () => {
     it('constructs a request to send to Sentry', async () => {
       expect.assertions(1);
       const transport = createTransport(transportOptions, req => {
-        expect(req.body).toEqual(serializeEnvelope(ERROR_ENVELOPE));
+        expect(req.body).toEqual(serializeEnvelope(ERROR_ENVELOPE, new TextEncoder()));
         return resolvedSyncPromise({});
       });
       await transport.send(ERROR_ENVELOPE);
@@ -46,7 +48,7 @@ describe('createTransport', () => {
       expect.assertions(2);
 
       const transport = createTransport(transportOptions, req => {
-        expect(req.body).toEqual(serializeEnvelope(ERROR_ENVELOPE));
+        expect(req.body).toEqual(serializeEnvelope(ERROR_ENVELOPE, new TextEncoder()));
         throw new Error();
       });
 
@@ -82,7 +84,10 @@ describe('createTransport', () => {
 
         const mockRecordDroppedEventCallback = jest.fn();
 
-        const transport = createTransport({ recordDroppedEvent: mockRecordDroppedEventCallback }, mockRequestExecutor);
+        const transport = createTransport(
+          { recordDroppedEvent: mockRecordDroppedEventCallback, textEncoder: new TextEncoder() },
+          mockRequestExecutor,
+        );
 
         return [transport, setTransportResponse, mockRequestExecutor, mockRecordDroppedEventCallback] as const;
       }

packages/core/test/mocks/client.ts

@@ -1,5 +1,6 @@
-import { ClientOptions, Event, Integration, Outcome, Session, Severity, SeverityLevel } from '@sentry/types';
+import { ClientOptions, Event, EventHint, Integration, Outcome, Session, Severity, SeverityLevel } from '@sentry/types';
 import { resolvedSyncPromise } from '@sentry/utils';
+import { TextEncoder } from 'util';
 
 import { BaseClient } from '../../src/baseclient';
 import { initAndBind } from '../../src/sdk';
@@ -9,9 +10,13 @@ export function getDefaultTestClientOptions(options: Partial<TestClientOptions>
   return {
     integrations: [],
     sendClientReports: true,
+    transportOptions: { textEncoder: new TextEncoder() },
     transport: () =>
       createTransport(
-        { recordDroppedEvent: () => undefined }, // noop
+        {
+          recordDroppedEvent: () => undefined,
+          textEncoder: new TextEncoder(),
+        }, // noop
         _ => resolvedSyncPromise({}),
       ),
     stackParser: () => [],
@@ -62,10 +67,10 @@ export class TestClient extends BaseClient<TestClientOptions> {
     return resolvedSyncPromise({ message, level });
   }
 
-  public sendEvent(event: Event): void {
+  public sendEvent(event: Event, hint?: EventHint): void {
     this.event = event;
     if (this._options.enableSend) {
-      super.sendEvent(event);
+      super.sendEvent(event, hint);
       return;
     }
     // eslint-disable-next-line @typescript-eslint/no-unused-expressions

packages/core/test/mocks/transport.ts

@@ -1,4 +1,5 @@
 import { SyncPromise } from '@sentry/utils';
+import { TextEncoder } from 'util';
 
 import { createTransport } from '../../src/transports/base';
 
@@ -10,7 +11,7 @@ export function makeFakeTransport(delay: number = 2000) {
   let sendCalled = 0;
   let sentCount = 0;
   const makeTransport = () =>
-    createTransport({ recordDroppedEvent: () => undefined }, () => {
+    createTransport({ recordDroppedEvent: () => undefined, textEncoder: new TextEncoder() }, () => {
       sendCalled += 1;
       return new SyncPromise(async res => {
         await sleep(delay);

packages/gatsby/test/integration.test.tsx

@@ -3,13 +3,16 @@ import { render } from '@testing-library/react';
 import { useEffect } from 'react';
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 import * as React from 'react';
+import { TextDecoder,TextEncoder } from 'util';
 
 import { onClientEntry } from '../gatsby-browser';
 import * as Sentry from '../src';
 
 beforeAll(() => {
   (global as any).__SENTRY_RELEASE__ = '683f3a6ab819d47d23abfca9a914c81f0524d35b';
   (global as any).__SENTRY_DSN__ = 'https://examplePublicKey@o0.ingest.sentry.io/0';
+  (global as any).TextEncoder = TextEncoder;
+  (global as any).TextDecoder = TextDecoder;
 });
 
 describe('useEffect', () => {