Add support for MSC4115 (#17104)

Co-authored-by: Andrew Morgan <[email protected]>

Author: richvdh

changelog.d/17104.feature

@@ -0,0 +1 @@
+Add support for MSC4115 (membership metadata on events).

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -92,8 +92,6 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##
 
 experimental_features:
-  # client-side support for partial state in /send_join responses
-  faster_joins: true
   # Enable support for polls
   msc3381_polls_enabled: true
   # Enable deleting device-specific notification settings stored in account data
@@ -105,6 +103,8 @@ experimental_features:
   # no UIA for x-signing upload for the first time
   msc3967_enabled: true
 
+  msc4115_membership_on_events: true
+
 server_notices:
   system_mxid_localpart: _server
   system_mxid_display_name: "Server Alert"

rust/src/events/internal_metadata.rs

@@ -20,8 +20,10 @@
 
 //! Implements the internal metadata class attached to events.
 //!
-//! The internal metadata is a bit like a `TypedDict`, in that it is stored as a
-//! JSON dict in the DB. Most events have zero, or only a few, of these keys
+//! The internal metadata is a bit like a `TypedDict`, in that most of
+//! it is stored as a JSON dict in the DB (the exceptions being `outlier`
+//! and `stream_ordering` which have their own columns in the database).
+//! Most events have zero, or only a few, of these keys
 //! set. Therefore, since we care more about memory size than performance here,
 //! we store these fields in a mapping.
 //!
@@ -234,6 +236,9 @@ impl EventInternalMetadata {
         self.clone()
     }
 
+    /// Get a dict holding the data stored in the `internal_metadata` column in the database.
+    ///
+    /// Note that `outlier` and `stream_ordering` are stored in separate columns so are not returned here.
     fn get_dict(&self, py: Python<'_>) -> PyResult<PyObject> {
         let dict = PyDict::new(py);
 

synapse/api/constants.py

@@ -234,6 +234,13 @@ class EventContentFields:
     TO_DEVICE_MSGID: Final = "org.matrix.msgid"
 
 
+class EventUnsignedContentFields:
+    """Fields found inside the 'unsigned' data on events"""
+
+    # Requesting user's membership, per MSC4115
+    MSC4115_MEMBERSHIP: Final = "io.element.msc4115.membership"
+
+
 class RoomTypes:
     """Understood values of the room_type field of m.room.create events."""
 

synapse/config/experimental.py

@@ -432,3 +432,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
                 "You cannot have MSC4108 both enabled and delegated at the same time",
                 ("experimental", "msc4108_delegation_endpoint"),
             )
+
+        self.msc4115_membership_on_events = experimental.get(
+            "msc4115_membership_on_events", False
+        )

synapse/events/utils.py

@@ -49,7 +49,7 @@
 from synapse.api.room_versions import RoomVersion
 from synapse.types import JsonDict, Requester
 
-from . import EventBase
+from . import EventBase, make_event_from_dict
 
 if TYPE_CHECKING:
     from synapse.handlers.relations import BundledAggregations
@@ -82,17 +82,14 @@ def prune_event(event: EventBase) -> EventBase:
     """
     pruned_event_dict = prune_event_dict(event.room_version, event.get_dict())
 
-    from . import make_event_from_dict
-
     pruned_event = make_event_from_dict(
         pruned_event_dict, event.room_version, event.internal_metadata.get_dict()
     )
 
-    # copy the internal fields
+    # Copy the bits of `internal_metadata` that aren't returned by `get_dict`
     pruned_event.internal_metadata.stream_ordering = (
         event.internal_metadata.stream_ordering
     )
-
     pruned_event.internal_metadata.outlier = event.internal_metadata.outlier
 
     # Mark the event as redacted
@@ -101,6 +98,29 @@ def prune_event(event: EventBase) -> EventBase:
     return pruned_event
 
 
+def clone_event(event: EventBase) -> EventBase:
+    """Take a copy of the event.
+
+    This is mostly useful because it does a *shallow* copy of the `unsigned` data,
+    which means it can then be updated without corrupting the in-memory cache. Note that
+    other properties of the event, such as `content`, are *not* (currently) copied here.
+    """
+    # XXX: We rely on at least one of `event.get_dict()` and `make_event_from_dict()`
+    #   making a copy of `unsigned`. Currently, both do, though I don't really know why.
+    #   Still, as long as they do, there's not much point doing yet another copy here.
+    new_event = make_event_from_dict(
+        event.get_dict(), event.room_version, event.internal_metadata.get_dict()
+    )
+
+    # Copy the bits of `internal_metadata` that aren't returned by `get_dict`.
+    new_event.internal_metadata.stream_ordering = (
+        event.internal_metadata.stream_ordering
+    )
+    new_event.internal_metadata.outlier = event.internal_metadata.outlier
+
+    return new_event
+
+
 def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDict:
     """Redacts the event_dict in the same way as `prune_event`, except it
     operates on dicts rather than event objects

synapse/handlers/admin.py

@@ -42,6 +42,7 @@ def __init__(self, hs: "HomeServer"):
         self._device_handler = hs.get_device_handler()
         self._storage_controllers = hs.get_storage_controllers()
         self._state_storage_controller = self._storage_controllers.state
+        self._hs_config = hs.config
         self._msc3866_enabled = hs.config.experimental.msc3866.enabled
 
     async def get_whois(self, user: UserID) -> JsonMapping:
@@ -217,7 +218,10 @@ async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") ->
                 )
 
                 events = await filter_events_for_client(
-                    self._storage_controllers, user_id, events
+                    self._storage_controllers,
+                    user_id,
+                    events,
+                    msc4115_membership_on_events=self._hs_config.experimental.msc4115_membership_on_events,
                 )
 
                 writer.write_events(room_id, events)

synapse/handlers/events.py

@@ -148,6 +148,7 @@ class EventHandler:
     def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastores().main
         self._storage_controllers = hs.get_storage_controllers()
+        self._config = hs.config
 
     async def get_event(
         self,
@@ -189,7 +190,11 @@ async def get_event(
         is_peeking = not is_user_in_room
 
         filtered = await filter_events_for_client(
-            self._storage_controllers, user.to_string(), [event], is_peeking=is_peeking
+            self._storage_controllers,
+            user.to_string(),
+            [event],
+            is_peeking=is_peeking,
+            msc4115_membership_on_events=self._config.experimental.msc4115_membership_on_events,
         )
 
         if not filtered:

synapse/handlers/initial_sync.py

@@ -221,7 +221,10 @@ async def handle_room(event: RoomsForUser) -> None:
                 ).addErrback(unwrapFirstError)
 
                 messages = await filter_events_for_client(
-                    self._storage_controllers, user_id, messages
+                    self._storage_controllers,
+                    user_id,
+                    messages,
+                    msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
                 )
 
                 start_token = now_token.copy_and_replace(StreamKeyType.ROOM, token)
@@ -380,6 +383,7 @@ async def _room_initial_sync_parted(
             requester.user.to_string(),
             messages,
             is_peeking=is_peeking,
+            msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
         )
 
         start_token = StreamToken.START.copy_and_replace(StreamKeyType.ROOM, token)
@@ -494,6 +498,7 @@ async def get_receipts() -> List[JsonMapping]:
             requester.user.to_string(),
             messages,
             is_peeking=is_peeking,
+            msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
         )
 
         start_token = now_token.copy_and_replace(StreamKeyType.ROOM, token)

synapse/handlers/pagination.py

@@ -623,6 +623,7 @@ async def get_messages(
                 user_id,
                 events,
                 is_peeking=(member_event_id is None),
+                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
             )
 
         # if after the filter applied there are no more events

synapse/handlers/relations.py

@@ -95,6 +95,7 @@ def __init__(self, hs: "HomeServer"):
         self._event_handler = hs.get_event_handler()
         self._event_serializer = hs.get_event_client_serializer()
         self._event_creation_handler = hs.get_event_creation_handler()
+        self._config = hs.config
 
     async def get_relations(
         self,
@@ -163,6 +164,7 @@ async def get_relations(
             user_id,
             events,
             is_peeking=(member_event_id is None),
+            msc4115_membership_on_events=self._config.experimental.msc4115_membership_on_events,
         )
 
         # The relations returned for the requested event do include their
@@ -608,6 +610,7 @@ async def get_threads(
             user_id,
             events,
             is_peeking=(member_event_id is None),
+            msc4115_membership_on_events=self._config.experimental.msc4115_membership_on_events,
         )
 
         aggregations = await self.get_bundled_aggregations(

synapse/handlers/room.py

@@ -1476,6 +1476,7 @@ async def filter_evts(events: List[EventBase]) -> List[EventBase]:
                 user.to_string(),
                 events,
                 is_peeking=is_peeking,
+                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
             )
 
         event = await self.store.get_event(

synapse/handlers/search.py

@@ -480,7 +480,10 @@ async def _search_by_rank(
         filtered_events = await search_filter.filter([r["event"] for r in results])
 
         events = await filter_events_for_client(
-            self._storage_controllers, user.to_string(), filtered_events
+            self._storage_controllers,
+            user.to_string(),
+            filtered_events,
+            msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
         )
 
         events.sort(key=lambda e: -rank_map[e.event_id])
@@ -579,7 +582,10 @@ async def _search_by_recent(
             filtered_events = await search_filter.filter([r["event"] for r in results])
 
             events = await filter_events_for_client(
-                self._storage_controllers, user.to_string(), filtered_events
+                self._storage_controllers,
+                user.to_string(),
+                filtered_events,
+                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
             )
 
             room_events.extend(events)
@@ -664,11 +670,17 @@ async def _calculate_event_contexts(
             )
 
             events_before = await filter_events_for_client(
-                self._storage_controllers, user.to_string(), res.events_before
+                self._storage_controllers,
+                user.to_string(),
+                res.events_before,
+                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
             )
 
             events_after = await filter_events_for_client(
-                self._storage_controllers, user.to_string(), res.events_after
+                self._storage_controllers,
+                user.to_string(),
+                res.events_after,
+                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
             )
 
             context: JsonDict = {

synapse/handlers/sync.py

@@ -596,6 +596,7 @@ async def _load_filtered_recents(
                     sync_config.user.to_string(),
                     recents,
                     always_include_ids=current_state_ids,
+                    msc4115_membership_on_events=self.hs_config.experimental.msc4115_membership_on_events,
                 )
                 log_kv({"recents_after_visibility_filtering": len(recents)})
             else:
@@ -681,6 +682,7 @@ async def _load_filtered_recents(
                     sync_config.user.to_string(),
                     loaded_recents,
                     always_include_ids=current_state_ids,
+                    msc4115_membership_on_events=self.hs_config.experimental.msc4115_membership_on_events,
                 )
 
                 loaded_recents = []

synapse/notifier.py

@@ -721,6 +721,7 @@ async def check_for_updates(
                         user.to_string(),
                         new_events,
                         is_peeking=is_peeking,
+                        msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
                     )
                 elif keyname == StreamKeyType.PRESENCE:
                     now = self.clock.time_msec()

synapse/push/mailer.py

@@ -529,7 +529,10 @@ async def _get_notif_vars(
         }
 
         the_events = await filter_events_for_client(
-            self._storage_controllers, user_id, results.events_before
+            self._storage_controllers,
+            user_id,
+            results.events_before,
+            msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
         )
         the_events.append(notif_event)
 

synapse/visibility.py

@@ -36,10 +36,15 @@
 
 import attr
 
-from synapse.api.constants import EventTypes, HistoryVisibility, Membership
+from synapse.api.constants import (
+    EventTypes,
+    EventUnsignedContentFields,
+    HistoryVisibility,
+    Membership,
+)
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
-from synapse.events.utils import prune_event
+from synapse.events.utils import clone_event, prune_event
 from synapse.logging.opentracing import trace
 from synapse.storage.controllers import StorageControllers
 from synapse.storage.databases.main import DataStore
@@ -77,6 +82,7 @@ async def filter_events_for_client(
     is_peeking: bool = False,
     always_include_ids: FrozenSet[str] = frozenset(),
     filter_send_to_client: bool = True,
+    msc4115_membership_on_events: bool = False,
 ) -> List[EventBase]:
     """
     Check which events a user is allowed to see. If the user can see the event but its
@@ -95,9 +101,12 @@ async def filter_events_for_client(
         filter_send_to_client: Whether we're checking an event that's going to be
             sent to a client. This might not always be the case since this function can
             also be called to check whether a user can see the state at a given point.
+        msc4115_membership_on_events: Whether to include the requesting user's
+            membership in the "unsigned" data, per MSC4115.
 
     Returns:
-        The filtered events.
+        The filtered events. If `msc4115_membership_on_events` is true, the `unsigned`
+        data is annotated with the membership state of `user_id` at each event.
     """
     # Filter out events that have been soft failed so that we don't relay them
     # to clients.
@@ -134,21 +143,54 @@ async def filter_events_for_client(
             )
 
     def allowed(event: EventBase) -> Optional[EventBase]:
-        return _check_client_allowed_to_see_event(
+        state_after_event = event_id_to_state.get(event.event_id)
+        filtered = _check_client_allowed_to_see_event(
             user_id=user_id,
             event=event,
             clock=storage.main.clock,
             filter_send_to_client=filter_send_to_client,
             sender_ignored=event.sender in ignore_list,
             always_include_ids=always_include_ids,
             retention_policy=retention_policies[room_id],
-            state=event_id_to_state.get(event.event_id),
+            state=state_after_event,
             is_peeking=is_peeking,
             sender_erased=erased_senders.get(event.sender, False),
         )
+        if filtered is None:
+            return None
+
+        if not msc4115_membership_on_events:
+            return filtered
+
+        # Annotate the event with the user's membership after the event.
+        #
+        # Normally we just look in `state_after_event`, but if the event is an outlier
+        # we won't have such a state. The only outliers that are returned here are the
+        # user's own membership event, so we can just inspect that.
+
+        user_membership_event: Optional[EventBase]
+        if event.type == EventTypes.Member and event.state_key == user_id:
+            user_membership_event = event
+        elif state_after_event is not None:
+            user_membership_event = state_after_event.get((EventTypes.Member, user_id))
+        else:
+            # unreachable!
+            raise Exception("Missing state for event that is not user's own membership")
+
+        user_membership = (
+            user_membership_event.membership
+            if user_membership_event
+            else Membership.LEAVE
+        )
 
-    # Check each event: gives an iterable of None or (a potentially modified)
-    # EventBase.
+        # Copy the event before updating the unsigned data: this shouldn't be persisted
+        # to the cache!
+        cloned = clone_event(filtered)
+        cloned.unsigned[EventUnsignedContentFields.MSC4115_MEMBERSHIP] = user_membership
+
+        return cloned
+
+    # Check each event: gives an iterable of None or (a modified) EventBase.
     filtered_events = map(allowed, events)
 
     # Turn it into a list and remove None entries before returning.
@@ -396,7 +438,13 @@ def _check_client_allowed_to_see_event(
 
 @attr.s(frozen=True, slots=True, auto_attribs=True)
 class _CheckMembershipReturn:
-    "Return value of _check_membership"
+    """Return value of `_check_membership`.
+
+    Attributes:
+        allowed: Whether the user should be allowed to see the event.
+        joined: Whether the user was joined to the room at the event.
+    """
+
     allowed: bool
     joined: bool
 
@@ -408,12 +456,7 @@ def _check_membership(
     state: StateMap[EventBase],
     is_peeking: bool,
 ) -> _CheckMembershipReturn:
-    """Check whether the user can see the event due to their membership
-
-    Returns:
-        True if they can, False if they can't, plus the membership of the user
-        at the event.
-    """
+    """Check whether the user can see the event due to their membership"""
     # If the event is the user's own membership event, use the 'most joined'
     # membership
     membership = None
@@ -435,7 +478,7 @@ def _check_membership(
         if membership == "leave" and (
             prev_membership == "join" or prev_membership == "invite"
         ):
-            return _CheckMembershipReturn(True, membership == Membership.JOIN)
+            return _CheckMembershipReturn(True, False)
 
         new_priority = MEMBERSHIP_PRIORITY.index(membership)
         old_priority = MEMBERSHIP_PRIORITY.index(prev_membership)

tests/events/test_utils.py

@@ -32,6 +32,7 @@
     PowerLevelsContent,
     SerializeEventConfig,
     _split_field,
+    clone_event,
     copy_and_fixup_power_levels_contents,
     maybe_upsert_event_field,
     prune_event,
@@ -611,6 +612,29 @@ def test_relations(self) -> None:
         )
 
 
+class CloneEventTestCase(stdlib_unittest.TestCase):
+    def test_unsigned_is_copied(self) -> None:
+        original = make_event_from_dict(
+            {
+                "type": "A",
+                "event_id": "$test:domain",
+                "unsigned": {"a": 1, "b": 2},
+            },
+            RoomVersions.V1,
+            {"txn_id": "txn"},
+        )
+        original.internal_metadata.stream_ordering = 1234
+        self.assertEqual(original.internal_metadata.stream_ordering, 1234)
+
+        cloned = clone_event(original)
+        cloned.unsigned["b"] = 3
+
+        self.assertEqual(original.unsigned, {"a": 1, "b": 2})
+        self.assertEqual(cloned.unsigned, {"a": 1, "b": 3})
+        self.assertEqual(cloned.internal_metadata.stream_ordering, 1234)
+        self.assertEqual(cloned.internal_metadata.txn_id, "txn")
+
+
 class SerializeEventTestCase(stdlib_unittest.TestCase):
     def serialize(self, ev: EventBase, fields: Optional[List[str]]) -> JsonDict:
         return serialize_event(

tests/rest/client/test_retention.py

@@ -163,7 +163,12 @@ def test_visibility(self) -> None:
         )
         self.assertEqual(2, len(events), "events retrieved from database")
         filtered_events = self.get_success(
-            filter_events_for_client(storage_controllers, self.user_id, events)
+            filter_events_for_client(
+                storage_controllers,
+                self.user_id,
+                events,
+                msc4115_membership_on_events=True,
+            )
         )
 
         # We should only get one event back.