Include the state before the event in roomserver output (#143)

* Include the state before the event in roomserver output

* Fix roomserver integration test

* Remove VisibilityEventIDs from the JSON

* More comments

* Remove spurious TODO

src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go

@@ -340,7 +340,8 @@ func main() {
 				"state_key":"@richvdh:matrix.org",
 				"type":"m.room.member"
 			},
-			"VisibilityEventIDs":null,
+			"StateBeforeRemovesEventIDs":["$1463671339126270PnVwC:matrix.org"],
+			"StateBeforeAddsEventIDs":null,
 			"LatestEventIDs":["$1463671339126270PnVwC:matrix.org"],
 			"AddsStateEventIDs":["$1463671337126266wrSBX:matrix.org", "$1463671339126270PnVwC:matrix.org"],
 			"RemovesStateEventIDs":null,

src/github.com/matrix-org/dendrite/roomserver/api/output.go

@@ -19,12 +19,17 @@ import (
 )
 
 // An OutputRoomEvent is written when the roomserver receives a new event.
+// It contains the full matrix room event and enough information for a
+// consumer to construct the current state of the room and the state before the
+// event.
+//
+// When we talk about state in a matrix room we are talking about the state
+// after a list of events. The current state is the state after the latest
+// event IDs in the room. The state before an event is the state after its
+// prev_events.
 type OutputRoomEvent struct {
 	// The JSON bytes of the event.
 	Event []byte
-	// The state event IDs needed to determine who can see this event.
-	// This can be used to tell which users to send the event to.
-	VisibilityEventIDs []string
 	// The latest events in the room after this event.
 	// This can be used to set the prev events for new events in the room.
 	// This also can be used to get the full current state after this event.
@@ -40,9 +45,27 @@ type OutputRoomEvent struct {
 	// This is used by consumers to check if they can safely update their
 	// current state using the delta supplied in AddsStateEventIDs and
 	// RemovesStateEventIDs.
+	//
 	// If the LastSentEventID doesn't match what they were expecting it to be
 	// they can use the LatestEventIDs to request the full current state.
 	LastSentEventID string
+	// The state event IDs that are part of the state at the event, but not
+	// part of the current state. Together with the StateBeforeRemovesEventIDs
+	// this can be used to construct the state before the event from the
+	// current state. The StateBeforeAddsEventIDs and StateBeforeRemovesEventIDs
+	// delta is applied after the AddsStateEventIDs and RemovesStateEventIDs.
+	//
+	// Consumers need to know the state at each event in order to determine
+	// which users and servers are allowed to see the event. This information
+	// is needed to apply the history visibility rules and to tell which
+	// servers we need to push events to over federation.
+	//
+	// The state is given as a delta against the current state because they are
+	// usually either the same state, or differ by just a couple of events.
+	StateBeforeAddsEventIDs []string
+	// The state event IDs that are part of the current state, but not part
+	// of the state at the event.
+	StateBeforeRemovesEventIDs []string
 }
 
 // UnmarshalJSON implements json.Unmarshaller
@@ -53,11 +76,12 @@ func (ore *OutputRoomEvent) UnmarshalJSON(data []byte) error {
 	// being base64 encoded which is the default for []byte.
 	var content struct {
 		Event                      *json.RawMessage
-		VisibilityEventIDs   []string
 		LatestEventIDs             []string
 		AddsStateEventIDs          []string
 		RemovesStateEventIDs       []string
 		LastSentEventID            string
+		StateBeforeAddsEventIDs    []string
+		StateBeforeRemovesEventIDs []string
 	}
 	if err := json.Unmarshal(data, &content); err != nil {
 		return err
@@ -65,11 +89,12 @@ func (ore *OutputRoomEvent) UnmarshalJSON(data []byte) error {
 	if content.Event != nil {
 		ore.Event = []byte(*content.Event)
 	}
-	ore.VisibilityEventIDs = content.VisibilityEventIDs
 	ore.LatestEventIDs = content.LatestEventIDs
 	ore.AddsStateEventIDs = content.AddsStateEventIDs
 	ore.RemovesStateEventIDs = content.RemovesStateEventIDs
 	ore.LastSentEventID = content.LastSentEventID
+	ore.StateBeforeAddsEventIDs = content.StateBeforeAddsEventIDs
+	ore.StateBeforeRemovesEventIDs = content.StateBeforeRemovesEventIDs
 	return nil
 }
 
@@ -82,18 +107,20 @@ func (ore OutputRoomEvent) MarshalJSON() ([]byte, error) {
 	event := json.RawMessage(ore.Event)
 	content := struct {
 		Event                      *json.RawMessage
-		VisibilityEventIDs   []string
 		LatestEventIDs             []string
 		AddsStateEventIDs          []string
 		RemovesStateEventIDs       []string
 		LastSentEventID            string
+		StateBeforeAddsEventIDs    []string
+		StateBeforeRemovesEventIDs []string
 	}{
 		Event:                      &event,
-		VisibilityEventIDs:   ore.VisibilityEventIDs,
 		LatestEventIDs:             ore.LatestEventIDs,
 		AddsStateEventIDs:          ore.AddsStateEventIDs,
 		RemovesStateEventIDs:       ore.RemovesStateEventIDs,
 		LastSentEventID:            ore.LastSentEventID,
+		StateBeforeAddsEventIDs:    ore.StateBeforeAddsEventIDs,
+		StateBeforeRemovesEventIDs: ore.StateBeforeRemovesEventIDs,
 	}
 	return json.Marshal(&content)
 }

src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go

@@ -20,6 +20,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 )
 
 // updateLatestEvents updates the list of latest events for this room in the database and writes the
@@ -110,6 +111,13 @@ func doUpdateLatestEvents(
 		return err
 	}
 
+	stateBeforeEventRemoves, stateBeforeEventAdds, err := state.DifferenceBetweeenStateSnapshots(
+		db, newStateNID, stateAtEvent.BeforeStateSnapshotNID,
+	)
+	if err != nil {
+		return err
+	}
+
 	// Send the event to the output logs.
 	// We do this inside the database transaction to ensure that we only mark an event as sent if we sent it.
 	// (n.b. this means that it's possible that the same event will be sent twice if the transaction fails but
@@ -118,7 +126,10 @@ func doUpdateLatestEvents(
 	// send the event asynchronously but we would need to ensure that 1) the events are written to the log in
 	// the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the
 	// necessary bookkeeping we'll keep the event sending synchronous for now.
-	if err = writeEvent(db, ow, lastEventIDSent, event, newLatest, removed, added); err != nil {
+	if err = writeEvent(
+		db, ow, lastEventIDSent, event, newLatest, removed, added,
+		stateBeforeEventRemoves, stateBeforeEventAdds,
+	); err != nil {
 		return err
 	}
 
@@ -170,6 +181,7 @@ func writeEvent(
 	db RoomEventDatabase, ow OutputRoomEventWriter, lastEventIDSent string,
 	event gomatrixserverlib.Event, latest []types.StateAtEventAndReference,
 	removed, added []types.StateEntry,
+	stateBeforeEventRemoves, stateBeforeEventAdds []types.StateEntry,
 ) error {
 
 	latestEventIDs := make([]string, len(latest))
@@ -190,6 +202,13 @@ func writeEvent(
 	for _, entry := range removed {
 		stateEventNIDs = append(stateEventNIDs, entry.EventNID)
 	}
+	for _, entry := range stateBeforeEventRemoves {
+		stateEventNIDs = append(stateEventNIDs, entry.EventNID)
+	}
+	for _, entry := range stateBeforeEventAdds {
+		stateEventNIDs = append(stateEventNIDs, entry.EventNID)
+	}
+	stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))]
 	eventIDMap, err := db.EventIDs(stateEventNIDs)
 	if err != nil {
 		return err
@@ -200,7 +219,17 @@ func writeEvent(
 	for _, entry := range removed {
 		ore.RemovesStateEventIDs = append(ore.RemovesStateEventIDs, eventIDMap[entry.EventNID])
 	}
-
+	for _, entry := range stateBeforeEventRemoves {
-	// TODO: Fill out VisibilityStateIDs
+		ore.StateBeforeRemovesEventIDs = append(ore.StateBeforeRemovesEventIDs, eventIDMap[entry.EventNID])
+	}
+	for _, entry := range stateBeforeEventAdds {
+		ore.StateBeforeAddsEventIDs = append(ore.StateBeforeAddsEventIDs, eventIDMap[entry.EventNID])
+	}
 	return ow.WriteOutputRoomEvent(ore)
 }
+
+type eventNIDSorter []types.EventNID
+
+func (s eventNIDSorter) Len() int           { return len(s) }
+func (s eventNIDSorter) Less(i, j int) bool { return s[i] < s[j] }
+func (s eventNIDSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }