feat(server): Support chunked formdata for Crashpad

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+**Features**:
+
+- Support chunked formdata keys for event payloads on the Minidump endpoint. Since crashpad has a limit for the length of custom attributes, the sentry event payload can be split up into `sentry__1`, `sentry__2`, etc. ([#721](https://github.com/getsentry/relay/pull/721))
+
 **Internal**:
 
 - Remove a temporary flag from attachment kafka messages indicating rate limited crash reports to Sentry. This is now enabled by default. ([#718](https://github.com/getsentry/relay/pull/718))

relay-server/src/actors/events.rs

@@ -33,7 +33,7 @@ use crate::actors::upstream::{SendRequest, UpstreamRelay, UpstreamRequestError};
 use crate::envelope::{self, AttachmentType, ContentType, Envelope, Item, ItemType};
 use crate::metrics::{RelayCounters, RelayHistograms, RelaySets, RelayTimers};
 use crate::service::ServerError;
-use crate::utils::{self, FormDataIter, FutureExt};
+use crate::utils::{self, ChunkedFormDataAggregator, FormDataIter, FutureExt};
 
 #[cfg(feature = "processing")]
 use {
@@ -468,14 +468,20 @@ impl EventProcessor {
 
     fn merge_formdata(&self, target: &mut SerdeValue, item: Item) {
         let payload = item.payload();
+        let mut aggregator = ChunkedFormDataAggregator::new();
+
         for entry in FormDataIter::new(&payload) {
             if entry.key() == "sentry" {
                 // Custom clients can submit longer payloads and should JSON encode event data into
                 // the optional `sentry` field.
                 match serde_json::from_str(entry.value()) {
                     Ok(event) => utils::merge_values(target, event),
-                    Err(_) => log::debug!("invalid json event payload in form data"),
+                    Err(_) => log::debug!("invalid json event payload in sentry form field"),
                 }
+            } else if let Some(index) = utils::get_sentry_chunk_index(entry.key(), "sentry__") {
+                // Electron SDK splits up long payloads into chunks starting at sentry__1 with an
+                // incrementing counter. Assemble these chunks here and then decode them below.
+                aggregator.insert(index, entry.value());
             } else if let Some(keys) = utils::get_sentry_entry_indexes(entry.key()) {
                 // Try to parse the nested form syntax `sentry[key][key]` This is required for the
                 // Breakpad client library, which only supports string values of up to 64
@@ -488,6 +494,13 @@ impl EventProcessor {
                 utils::update_nested_value(target, &["extra", entry.key()], entry.value());
             }
         }
+
+        if !aggregator.is_empty() {
+            match serde_json::from_str(&aggregator.join()) {
+                Ok(event) => utils::merge_values(target, event),
+                Err(_) => log::debug!("invalid json event payload in sentry__* form fields"),
+            }
+        }
     }
 
     fn extract_attached_event(

relay-server/src/endpoints/common.rs

@@ -253,6 +253,11 @@ pub fn event_id_from_formdata(data: &[u8]) -> Result<Option<EventId>, BadStoreRe
 ///  2. The `__sentry-event` event attachment.
 ///  3. The `sentry` JSON payload.
 ///  4. The `sentry[event_id]` formdata key.
+///
+/// # Limitations
+///
+/// Extracting the event id from chunked formdata fields on the Minidump endpoint (`sentry__1`,
+/// `sentry__2`, ...) is not supported. In this case, `None` is returned.
 pub fn event_id_from_items(items: &Items) -> Result<Option<EventId>, BadStoreRequest> {
     if let Some(item) = items.iter().find(|item| item.ty() == ItemType::Event) {
         if let Some(event_id) = event_id_from_json(&item.payload())? {

relay-server/src/utils/param_parser.rs

@@ -93,6 +93,49 @@ pub fn get_sentry_entry_indexes(param_name: &str) -> Option<Vec<&str>> {
     }
 }
 
+/// Extracts the chunk index of a key with the given prefix.
+///
+/// Electron SDK splits up long payloads into chunks starting at sentry__1 with an
+/// incrementing counter. Assemble these chunks here and then decode them below.
+pub fn get_sentry_chunk_index(key: &str, prefix: &str) -> Option<usize> {
+    key.strip_prefix(prefix).and_then(|rest| rest.parse().ok())
+}
+
+/// Aggregates slices of strings in random order.
+#[derive(Clone, Debug, Default)]
+pub struct ChunkedFormDataAggregator<'a> {
+    parts: Vec<&'a str>,
+}
+
+impl<'a> ChunkedFormDataAggregator<'a> {
+    /// Creates a new empty aggregator.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Adds a part with the given index.
+    ///
+    /// Fills up unpopulated indexes with empty strings, if there are holes between the last index
+    /// and this one. This effectively skips them when calling `join` in the end.
+    pub fn insert(&mut self, index: usize, value: &'a str) {
+        if index >= self.parts.len() {
+            self.parts.resize(index + 1, "");
+        }
+
+        self.parts[index] = value;
+    }
+
+    /// Returns `true` if no parts have been added.
+    pub fn is_empty(&self) -> bool {
+        self.parts.is_empty()
+    }
+
+    /// Returns the string consisting of all parts.
+    pub fn join(&self) -> String {
+        self.parts.join("")
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -192,4 +235,72 @@ mod tests {
        ⋮}
         "###);
     }
+
+    #[test]
+    fn test_chunk_index() {
+        assert_eq!(get_sentry_chunk_index("sentry__0", "sentry__"), Some(0));
+        assert_eq!(get_sentry_chunk_index("sentry__1", "sentry__"), Some(1));
+
+        assert_eq!(get_sentry_chunk_index("foo__0", "sentry__"), None);
+        assert_eq!(get_sentry_chunk_index("sentry__", "sentry__"), None);
+        assert_eq!(get_sentry_chunk_index("sentry__-1", "sentry__"), None);
+        assert_eq!(get_sentry_chunk_index("sentry__xx", "sentry__"), None);
+    }
+
+    #[test]
+    fn test_aggregator_empty() {
+        let aggregator = ChunkedFormDataAggregator::new();
+        assert!(aggregator.is_empty());
+        assert_eq!(aggregator.join(), "");
+    }
+
+    #[test]
+    fn test_aggregator_base_0() {
+        let mut aggregator = ChunkedFormDataAggregator::new();
+        aggregator.insert(0, "hello,");
+        aggregator.insert(1, " world");
+
+        assert!(!aggregator.is_empty());
+        assert_eq!(aggregator.join(), "hello, world");
+    }
+
+    #[test]
+    fn test_aggregator_base_1() {
+        let mut aggregator = ChunkedFormDataAggregator::new();
+        aggregator.insert(1, "hello,");
+        aggregator.insert(2, " world");
+
+        assert!(!aggregator.is_empty());
+        assert_eq!(aggregator.join(), "hello, world");
+    }
+
+    #[test]
+    fn test_aggregator_holes() {
+        let mut aggregator = ChunkedFormDataAggregator::new();
+        aggregator.insert(0, "hello,");
+        aggregator.insert(3, " world");
+
+        assert!(!aggregator.is_empty());
+        assert_eq!(aggregator.join(), "hello, world");
+    }
+
+    #[test]
+    fn test_aggregator_reversed() {
+        let mut aggregator = ChunkedFormDataAggregator::new();
+        aggregator.insert(1, " world");
+        aggregator.insert(0, "hello,");
+
+        assert!(!aggregator.is_empty());
+        assert_eq!(aggregator.join(), "hello, world");
+    }
+
+    #[test]
+    fn test_aggregator_override() {
+        let mut aggregator = ChunkedFormDataAggregator::new();
+        aggregator.insert(0, "hello,");
+        aggregator.insert(0, "bye");
+
+        assert!(!aggregator.is_empty());
+        assert_eq!(aggregator.join(), "bye");
+    }
 }

tests/integration/test_minidump.py

@@ -188,6 +188,41 @@ def test_minidump_sentry_json(mini_sentry, relay):
     assert event_item["user"]["id"] == "123"
 
 
+def test_minidump_sentry_json_chunked(mini_sentry, relay):
+    project_id = 42
+    relay = relay(mini_sentry)
+    mini_sentry.project_configs[project_id] = mini_sentry.full_project_config()
+
+    attachments = [
+        (MINIDUMP_ATTACHMENT_NAME, "minidump.dmp", "MDMP content"),
+    ]
+
+    event_json = '{"event_id":"2dd132e467174db48dbaddabd3cbed57","user":{"id":"123"}}'
+    params = [
+        ("sentry__1", event_json[:30]),
+        ("sentry__2", event_json[30:]),
+    ]
+
+    response = relay.send_minidump(
+        project_id=project_id, files=attachments, params=params
+    )
+    envelope = mini_sentry.captured_events.get(timeout=1)
+
+    assert envelope
+    assert_only_minidump(envelope)
+
+    # With chunked JSON payloads, inferring event ids is not supported.
+    # The event id is randomized by Relay and overwritten.
+    event_id = UUID(response.text.strip()).hex
+    assert event_id != "2dd132e467174db48dbaddabd3cbed57"
+    assert envelope.headers.get("event_id") == event_id
+
+    # Check that event payload is applied
+    event_item = envelope.get_event()
+    assert event_item["event_id"] == event_id
+    assert event_item["user"]["id"] == "123"
+
+
 def test_minidump_invalid_json(mini_sentry, relay):
     project_id = 42
     relay = relay(mini_sentry)