feat: change how request bodies are dealt with

Author: untitaker

docs/config.md

@@ -49,3 +49,11 @@
   The provided list must contain actual classes, "string identifiers" are not
   supported. Subclass hierarchies are respected, for example, setting
   ``ignore_errors=[BaseException]`` will ignore all exceptions.
+- ``request_bodies`` (default ``"medium"``): One of
+  ``always|medium|small|never``. Whether to send the request body with each event.
+
+  - ``always``: Always send the request body. It is still omitted if it is not
+    JSON or formdata.
+  - ``medium``: Send "medium-large" request bodies but omit bigger ones.
+  - ``small``: Send "small" request bodies but omit bigger ones.
+  - ``never``: Never touch the request body.

sentry_sdk/client.py

@@ -58,6 +58,14 @@ def __init__(self, dsn=None, *args, **kwargs):
         for integration in integrations:
             integration(self)
 
+        request_bodies = ("always", "never", "small", "medium")
+        if options["request_bodies"] not in request_bodies:
+            raise ValueError(
+                "Invalid value for request_bodies. Must be one of {}".format(
+                    request_bodies
+                )
+            )
+
         atexit.register(self.close)
 
     @property

sentry_sdk/consts.py

@@ -19,6 +19,7 @@
     "http_proxy": None,
     "https_proxy": None,
     "ignore_errors": (),
+    "request_bodies": "medium",
 }
 
 SDK_INFO = {"name": "sentry-python", "version": VERSION}

sentry_sdk/integrations/_wsgi.py

@@ -1,5 +1,4 @@
 import json
-import base64
 
 from sentry_sdk.hub import _should_send_default_pii
 from sentry_sdk.stripping import AnnotatedValue
@@ -38,7 +37,7 @@ class RequestExtractor(object):
     def __init__(self, request):
         self.request = request
 
-    def extract_into_event(self, event):
+    def extract_into_event(self, event, client_options):
         if "request" in event:
             return
 
@@ -62,40 +61,48 @@ def extract_into_event(self, event):
                 not in ("set-cookie", "cookie", "authentication")
             }
 
-        if self.form or self.files:
+        bodies = client_options.get("request_bodies")
+        if (
+            bodies == "never"
+            or (bodies == "small" and self.content_length > 10 ** 3)
+            or (bodies == "medium" and self.content_length > 10 ** 6)
+        ):
+            data = AnnotatedValue(
+                "",
+                {
+                    "rem": [["!config", "x", 0, self.content_length]],
+                    "len": self.content_length,
+                },
+            )
+        elif self.form or self.files:
             data = dict(self.form.items())
             for k, v in self.files.items():
+                size = self.size_of_file(v)
                 data[k] = AnnotatedValue(
-                    "",
-                    {"len": self.size_of_file(v), "rem": [["!filecontent", "x", 0, 0]]},
+                    "", {"len": size, "rem": [["!filecontent", "x", 0, size]]}
                 )
 
-            if self.files or self.form_is_multipart:
-                ct = "multipart"
-            else:
-                ct = "urlencoded"
-            repr = "structured"
         elif self.json is not None:
             data = self.json
-            ct = "json"
-            repr = "structured"
         elif self.raw_data:
-            data = self.raw_data
-
-            try:
-                if isinstance(data, bytes):
-                    data = data.decode("utf-8")
-                ct = "plain"
-                repr = "other"
-            except UnicodeDecodeError:
-                ct = "bytes"
-                repr = "base64"
-                data = base64.b64encode(data).decode("ascii")
+            data = AnnotatedValue(
+                "",
+                {
+                    "rem": [["!rawbody", "x", 0, self.content_length]],
+                    "len": self.content_length,
+                },
+            )
         else:
             return
 
         request_info["data"] = data
-        request_info["data_info"] = {"ct": ct, "repr": repr}
+
+    @property
+    def content_length(self):
+        try:
+            return int(self.env.get("CONTENT_LENGTH", 0))
+        except ValueError:
+            return 0
 
     @property
     def url(self):

sentry_sdk/integrations/django.py

@@ -55,9 +55,13 @@ def sentry_patched_get_response(self, request):
         signals.got_request_exception.connect(_got_request_exception)
 
     def _make_event_processor(self, request):
+        client_options = get_current_hub().client.options
+
         def processor(event):
             with _internal_exceptions():
-                DjangoRequestExtractor(request).extract_into_event(event)
+                DjangoRequestExtractor(request).extract_into_event(
+                    event, client_options
+                )
 
             if _should_send_default_pii():
                 with _internal_exceptions():

sentry_sdk/integrations/flask.py

@@ -46,6 +46,7 @@ def _capture_exception(sender, exception, **kwargs):
 def _make_event_processor():
     request = getattr(_request_ctx_stack.top, "request", None)
     app = getattr(_app_ctx_stack.top, "app", None)
+    client_options = get_current_hub().client.options
 
     def event_processor(event):
         if request:
@@ -54,7 +55,7 @@ def event_processor(event):
                     event["transaction"] = request.url_rule.endpoint
 
             with _internal_exceptions():
-                FlaskRequestExtractor(request).extract_into_event(event)
+                FlaskRequestExtractor(request).extract_into_event(event, client_options)
 
         if _should_send_default_pii():
             with _internal_exceptions():

tests/integrations/flask/test_flask.py

@@ -182,10 +182,9 @@ def index():
         "": {"len": 2000, "rem": [["!len", "x", 509, 512]]}
     }
     assert len(event["request"]["data"]["foo"]["bar"]) == 512
-    assert event["request"]["data_info"] == {"ct": "json", "repr": "structured"}
 
 
-def test_flask_large_formdata_request(sentry_init, capture_events, app):
+def test_flask_medium_formdata_request(sentry_init, capture_events, app):
     sentry_init(integrations=[flask_sentry.FlaskIntegration()])
 
     data = {"foo": "a" * 2000}
@@ -209,12 +208,11 @@ def index():
         "": {"len": 2000, "rem": [["!len", "x", 509, 512]]}
     }
     assert len(event["request"]["data"]["foo"]) == 512
-    assert event["request"]["data_info"] == {"ct": "urlencoded", "repr": "structured"}
 
 
 @pytest.mark.parametrize("input_char", [u"a", b"a"])
-def test_flask_large_text_request(sentry_init, input_char, capture_events, app):
-    sentry_init(integrations=[flask_sentry.FlaskIntegration()])
+def test_flask_too_large_raw_request(sentry_init, input_char, capture_events, app):
+    sentry_init(integrations=[flask_sentry.FlaskIntegration()], request_bodies="small")
 
     data = input_char * 2000
 
@@ -237,41 +235,13 @@ def index():
 
     event, = events
     assert event[""]["request"]["data"] == {
-        "": {"len": 2000, "rem": [["!len", "x", 509, 512]]}
-    }
-    assert len(event["request"]["data"]) == 512
-    assert event["request"]["data_info"] == {"ct": "plain", "repr": "other"}
-
-
-def test_flask_large_bytes_request(sentry_init, capture_events, app):
-    sentry_init(integrations=[flask_sentry.FlaskIntegration()])
-
-    data = b"\xc3" * 2000
-
-    @app.route("/", methods=["POST"])
-    def index():
-        assert not request.form
-        assert request.data == data
-        assert not request.json
-        capture_message("hi")
-        return "ok"
-
-    events = capture_events()
-
-    client = app.test_client()
-    response = client.post("/", data=data)
-    assert response.status_code == 200
-
-    event, = events
-    assert event[""]["request"]["data"] == {
-        "": {"len": 2668, "rem": [["!len", "x", 509, 512]]}
+        "": {"len": 2000, "rem": [["!config", "x", 0, 2000]]}
     }
-    assert len(event["request"]["data"]) == 512
-    assert event["request"]["data_info"] == {"ct": "bytes", "repr": "base64"}
+    assert not event["request"]["data"]
 
 
 def test_flask_files_and_form(sentry_init, capture_events, app):
-    sentry_init(integrations=[flask_sentry.FlaskIntegration()])
+    sentry_init(integrations=[flask_sentry.FlaskIntegration()], request_bodies="always")
 
     data = {"foo": "a" * 2000, "file": (BytesIO(b"hello"), "hello.txt")}