Sync fixes (#1709)

* omitempty some fields in sync

* Add a few more

* Don't send push rules over and over again in incremental sync

* Further tweaks

syncapi/streams/stream_accountdata.go

@@ -82,11 +82,6 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 		return from
 	}
 
-	if len(dataTypes) == 0 {
-		// TODO: this fixes the sytest but is it the right thing to do?
-		dataTypes[""] = []string{"m.push_rules"}
-	}
-
 	// Iterate over the rooms
 	for roomID, dataTypes := range dataTypes {
 		// Request the missing data from the database
@@ -114,7 +109,10 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 				}
 			} else {
 				if roomData, ok := dataRes.RoomAccountData[roomID][dataType]; ok {
-					joinData := req.Response.Rooms.Join[roomID]
+					joinData := *types.NewJoinResponse()
+					if existing, ok := req.Response.Rooms.Join[roomID]; ok {
+						joinData = existing
+					}
 					joinData.AccountData.Events = append(
 						joinData.AccountData.Events,
 						gomatrixserverlib.ClientEvent{

syncapi/streams/stream_pdu.go

@@ -173,22 +173,23 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	switch delta.Membership {
 	case gomatrixserverlib.Join:
 		jr := types.NewJoinResponse()
-
 		jr.Timeline.PrevBatch = &prevBatch
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 		jr.Timeline.Limited = limited
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
 		res.Rooms.Join[delta.RoomID] = *jr
+
 	case gomatrixserverlib.Peek:
 		jr := types.NewJoinResponse()
-
 		jr.Timeline.PrevBatch = &prevBatch
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 		jr.Timeline.Limited = limited
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
 		res.Rooms.Peek[delta.RoomID] = *jr
+
 	case gomatrixserverlib.Leave:
 		fallthrough // transitions to leave are the same as ban
+
 	case gomatrixserverlib.Ban:
 		// TODO: recentEvents may contain events that this user is not allowed to see because they are
 		//       no longer in the room.

syncapi/streams/stream_receipt.go

@@ -59,7 +59,10 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 	}
 
 	for roomID, receipts := range receiptsByRoom {
-		jr := req.Response.Rooms.Join[roomID]
+		jr := *types.NewJoinResponse()
+		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
+			jr = existing
+		}
 		var ok bool
 
 		ev := gomatrixserverlib.ClientEvent{

syncapi/streams/stream_typing.go

@@ -32,7 +32,10 @@ func (p *TypingStreamProvider) IncrementalSync(
 			continue
 		}
 
-		jr := req.Response.Rooms.Join[roomID]
+		jr := *types.NewJoinResponse()
+		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
+			jr = existing
+		}
 
 		if users, updated := p.EDUCache.GetTypingUsersIfUpdatedAfter(
 			roomID, int64(from),

syncapi/types/types.go

@@ -360,11 +360,11 @@ type PrevEventRef struct {
 type Response struct {
 	NextBatch   StreamingToken `json:"next_batch"`
 	AccountData struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
+		Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
-	} `json:"account_data,omitempty"`
+	} `json:"account_data"`
 	Presence struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
+		Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
-	} `json:"presence,omitempty"`
+	} `json:"presence"`
 	Rooms struct {
 		Join   map[string]JoinResponse   `json:"join"`
 		Peek   map[string]JoinResponse   `json:"peek"`
@@ -372,13 +372,13 @@ type Response struct {
 		Leave  map[string]LeaveResponse  `json:"leave"`
 	} `json:"rooms"`
 	ToDevice struct {
-		Events []gomatrixserverlib.SendToDeviceEvent `json:"events"`
+		Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"`
 	} `json:"to_device"`
 	DeviceLists struct {
 		Changed []string `json:"changed,omitempty"`
 		Left    []string `json:"left,omitempty"`
-	} `json:"device_lists,omitempty"`
+	} `json:"device_lists"`
-	DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count"`
+	DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
 }
 
 // NewResponse creates an empty response with initialised maps.
@@ -386,19 +386,19 @@ func NewResponse() *Response {
 	res := Response{}
 	// Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section,
 	// so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors.
-	res.Rooms.Join = make(map[string]JoinResponse)
+	res.Rooms.Join = map[string]JoinResponse{}
-	res.Rooms.Peek = make(map[string]JoinResponse)
+	res.Rooms.Peek = map[string]JoinResponse{}
-	res.Rooms.Invite = make(map[string]InviteResponse)
+	res.Rooms.Invite = map[string]InviteResponse{}
-	res.Rooms.Leave = make(map[string]LeaveResponse)
+	res.Rooms.Leave = map[string]LeaveResponse{}
 
 	// Also pre-intialise empty slices or else we'll insert 'null' instead of '[]' for the value.
 	// TODO: We really shouldn't have to do all this to coerce encoding/json to Do The Right Thing. We should
 	//       really be using our own Marshal/Unmarshal implementations otherwise this may prove to be a CPU bottleneck.
 	//       This also applies to NewJoinResponse, NewInviteResponse and NewLeaveResponse.
-	res.AccountData.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.AccountData.Events = []gomatrixserverlib.ClientEvent{}
-	res.Presence.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.Presence.Events = []gomatrixserverlib.ClientEvent{}
-	res.ToDevice.Events = make([]gomatrixserverlib.SendToDeviceEvent, 0)
+	res.ToDevice.Events = []gomatrixserverlib.SendToDeviceEvent{}
-	res.DeviceListsOTKCount = make(map[string]int)
+	res.DeviceListsOTKCount = map[string]int{}
 
 	return &res
 }
@@ -435,10 +435,10 @@ type JoinResponse struct {
 // NewJoinResponse creates an empty response with initialised arrays.
 func NewJoinResponse() *JoinResponse {
 	res := JoinResponse{}
-	res.State.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.State.Events = []gomatrixserverlib.ClientEvent{}
-	res.Timeline.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.Timeline.Events = []gomatrixserverlib.ClientEvent{}
-	res.Ephemeral.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.Ephemeral.Events = []gomatrixserverlib.ClientEvent{}
-	res.AccountData.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.AccountData.Events = []gomatrixserverlib.ClientEvent{}
 	return &res
 }
 
@@ -487,8 +487,8 @@ type LeaveResponse struct {
 // NewLeaveResponse creates an empty response with initialised arrays.
 func NewLeaveResponse() *LeaveResponse {
 	res := LeaveResponse{}
-	res.State.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.State.Events = []gomatrixserverlib.ClientEvent{}
-	res.Timeline.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.Timeline.Events = []gomatrixserverlib.ClientEvent{}
 	return &res
 }