sytest: Make 'Inbound federation can backfill events' pass (#1051)

* sytest: Make 'Inbound federation can backfill events' pass

This breaks 'Outbound federation can backfill events' because now
we are returning the right number of events, which the previous
test was relying on.

Previously, /messages was backfilling the membership event, causing
the test to pass. Now we are no longer backfilling the membership
event due to the change in this commit, causing the test to fail.

The test should instead be returning the membership event locally
from synacpis database, but it doesn't do it fast enough, resulting
in a no-op /sync response with a next_batch=s0_0 which will never
pick up the local membership event when it rolls in. The test
does attempt to retry, but doesn't take the new next_batch=s1_0
resulting in it missing from the /messages response.

* Linting
This commit is contained in:
Kegsay 2020-05-20 16:04:31 +01:00 committed by GitHub
parent 260e69d138
commit 1414922026
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 86 additions and 45 deletions

View File

@ -70,7 +70,12 @@ func Backfill(
// Populate the request. // Populate the request.
req := api.QueryBackfillRequest{ req := api.QueryBackfillRequest{
RoomID: roomID, RoomID: roomID,
EarliestEventsIDs: eIDs, // we don't know who the successors are for these events, which won't
// be a problem because we don't use that information when servicing /backfill requests,
// only when making them. TODO: Think of a better API shape
BackwardsExtremities: map[string][]string{
"": eIDs,
},
ServerName: request.Origin(), ServerName: request.Origin(),
} }
if req.Limit, err = strconv.Atoi(limit); err != nil { if req.Limit, err = strconv.Atoi(limit); err != nil {
@ -105,6 +110,12 @@ func Backfill(
eventJSONs = append(eventJSONs, e.JSON()) eventJSONs = append(eventJSONs, e.JSON())
} }
// sytest wants these in reversed order, similar to /messages, so reverse them now.
for i := len(eventJSONs)/2 - 1; i >= 0; i-- {
opp := len(eventJSONs) - 1 - i
eventJSONs[i], eventJSONs[opp] = eventJSONs[opp], eventJSONs[i]
}
txn := gomatrixserverlib.Transaction{ txn := gomatrixserverlib.Transaction{
Origin: cfg.Matrix.ServerName, Origin: cfg.Matrix.ServerName,
PDUs: eventJSONs, PDUs: eventJSONs,

View File

@ -21,6 +21,7 @@ import (
commonHTTP "github.com/matrix-org/dendrite/common/http" commonHTTP "github.com/matrix-org/dendrite/common/http"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
opentracing "github.com/opentracing/opentracing-go" opentracing "github.com/opentracing/opentracing-go"
) )
@ -228,14 +229,24 @@ type QueryStateAndAuthChainResponse struct {
type QueryBackfillRequest struct { type QueryBackfillRequest struct {
// The room to backfill // The room to backfill
RoomID string `json:"room_id"` RoomID string `json:"room_id"`
// Events to start paginating from. // A map of backwards extremity event ID to a list of its prev_event IDs.
EarliestEventsIDs []string `json:"earliest_event_ids"` BackwardsExtremities map[string][]string `json:"backwards_extremities"`
// The maximum number of events to retrieve. // The maximum number of events to retrieve.
Limit int `json:"limit"` Limit int `json:"limit"`
// The server interested in the events. // The server interested in the events.
ServerName gomatrixserverlib.ServerName `json:"server_name"` ServerName gomatrixserverlib.ServerName `json:"server_name"`
} }
// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order.
func (r *QueryBackfillRequest) PrevEventIDs() []string {
var prevEventIDs []string
for _, pes := range r.BackwardsExtremities {
prevEventIDs = append(prevEventIDs, pes...)
}
prevEventIDs = util.UniqueStrings(prevEventIDs)
return prevEventIDs
}
// QueryBackfillResponse is a response to QueryBackfill. // QueryBackfillResponse is a response to QueryBackfill.
type QueryBackfillResponse struct { type QueryBackfillResponse struct {
// Missing events, arbritrary order. // Missing events, arbritrary order.

View File

@ -495,14 +495,8 @@ func (r *RoomserverInternalAPI) QueryBackfill(
// defines the highest number of elements in the map below. // defines the highest number of elements in the map below.
visited := make(map[string]bool, request.Limit) visited := make(map[string]bool, request.Limit)
// The provided event IDs have already been seen by the request's emitter, // this will include these events which is what we want
// and will be retrieved anyway, so there's no need to care about them if front = request.PrevEventIDs()
// they appear in our exploration of the event tree.
for _, id := range request.EarliestEventsIDs {
visited[id] = true
}
front = request.EarliestEventsIDs
// Scan the event tree for events to send back. // Scan the event tree for events to send back.
resultNIDs, err := r.scanEventTree(ctx, front, visited, request.Limit, request.ServerName) resultNIDs, err := r.scanEventTree(ctx, front, visited, request.Limit, request.ServerName)
@ -534,7 +528,7 @@ func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *
if err != nil { if err != nil {
return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err)
} }
requester := newBackfillRequester(r.DB, r.FedClient, r.ServerName) requester := newBackfillRequester(r.DB, r.FedClient, r.ServerName, req.BackwardsExtremities)
// Request 100 items regardless of what the query asks for. // Request 100 items regardless of what the query asks for.
// We don't want to go much higher than this. // We don't want to go much higher than this.
// We can't honour exactly the limit as some sytests rely on requesting more for tests to pass // We can't honour exactly the limit as some sytests rely on requesting more for tests to pass
@ -542,7 +536,7 @@ func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *
// Specifically the test "Outbound federation can backfill events" // Specifically the test "Outbound federation can backfill events"
events, err := gomatrixserverlib.RequestBackfill( events, err := gomatrixserverlib.RequestBackfill(
ctx, requester, ctx, requester,
r.KeyRing, req.RoomID, roomVer, req.EarliestEventsIDs, 100) r.KeyRing, req.RoomID, roomVer, req.PrevEventIDs(), 100)
if err != nil { if err != nil {
return err return err
} }

View File

@ -16,6 +16,7 @@ type backfillRequester struct {
db storage.Database db storage.Database
fedClient *gomatrixserverlib.FederationClient fedClient *gomatrixserverlib.FederationClient
thisServer gomatrixserverlib.ServerName thisServer gomatrixserverlib.ServerName
bwExtrems map[string][]string
// per-request state // per-request state
servers []gomatrixserverlib.ServerName servers []gomatrixserverlib.ServerName
@ -23,13 +24,14 @@ type backfillRequester struct {
eventIDMap map[string]gomatrixserverlib.Event eventIDMap map[string]gomatrixserverlib.Event
} }
func newBackfillRequester(db storage.Database, fedClient *gomatrixserverlib.FederationClient, thisServer gomatrixserverlib.ServerName) *backfillRequester { func newBackfillRequester(db storage.Database, fedClient *gomatrixserverlib.FederationClient, thisServer gomatrixserverlib.ServerName, bwExtrems map[string][]string) *backfillRequester {
return &backfillRequester{ return &backfillRequester{
db: db, db: db,
fedClient: fedClient, fedClient: fedClient,
thisServer: thisServer, thisServer: thisServer,
eventIDToBeforeStateIDs: make(map[string][]string), eventIDToBeforeStateIDs: make(map[string][]string),
eventIDMap: make(map[string]gomatrixserverlib.Event), eventIDMap: make(map[string]gomatrixserverlib.Event),
bwExtrems: bwExtrems,
} }
} }
@ -160,26 +162,44 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr
// It returns a list of servers which can be queried for backfill requests. These servers // It returns a list of servers which can be queried for backfill requests. These servers
// will be servers that are in the room already. The entries at the beginning are preferred servers // will be servers that are in the room already. The entries at the beginning are preferred servers
// and will be tried first. An empty list will fail the request. // and will be tried first. An empty list will fail the request.
func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID string) (servers []gomatrixserverlib.ServerName) { func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID string) []gomatrixserverlib.ServerName {
// eventID will be a prev_event ID of a backwards extremity, meaning we will not have a database entry for it. Instead, use
// its successor, so look it up.
successor := ""
FindSuccessor:
for sucID, prevEventIDs := range b.bwExtrems {
for _, pe := range prevEventIDs {
if pe == eventID {
successor = sucID
break FindSuccessor
}
}
}
if successor == "" {
logrus.WithField("event_id", eventID).Error("ServersAtEvent: failed to find successor of this event to determine room state")
return nil
}
eventID = successor
// getMembershipsBeforeEventNID requires a NID, so retrieving the NID for // getMembershipsBeforeEventNID requires a NID, so retrieving the NID for
// the event is necessary. // the event is necessary.
NIDs, err := b.db.EventNIDs(ctx, []string{eventID}) NIDs, err := b.db.EventNIDs(ctx, []string{eventID})
if err != nil { if err != nil {
logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get event NID for event") logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get event NID for event")
return return nil
} }
stateEntries, err := stateBeforeEvent(ctx, b.db, NIDs[eventID]) stateEntries, err := stateBeforeEvent(ctx, b.db, NIDs[eventID])
if err != nil { if err != nil {
logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event") logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event")
return return nil
} }
// possibly return all joined servers depending on history visiblity // possibly return all joined servers depending on history visiblity
memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries) memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries)
if err != nil { if err != nil {
logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules") logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules")
return return nil
} }
logrus.Infof("ServersAtEvent including %d current events from history visibility", len(memberEventsFromVis)) logrus.Infof("ServersAtEvent including %d current events from history visibility", len(memberEventsFromVis))
@ -189,7 +209,7 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID
memberEvents, err := getMembershipsAtState(ctx, b.db, stateEntries, true) memberEvents, err := getMembershipsAtState(ctx, b.db, stateEntries, true)
if err != nil { if err != nil {
logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event") logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event")
return return nil
} }
memberEvents = append(memberEvents, memberEventsFromVis...) memberEvents = append(memberEvents, memberEventsFromVis...)
@ -198,6 +218,7 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID
for _, event := range memberEvents { for _, event := range memberEvents {
serverSet[event.Origin()] = true serverSet[event.Origin()] = true
} }
var servers []gomatrixserverlib.ServerName
for server := range serverSet { for server := range serverSet {
if server == b.thisServer { if server == b.thisServer {
continue continue
@ -205,7 +226,7 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID
servers = append(servers, server) servers = append(servers, server)
} }
b.servers = servers b.servers = servers
return return servers
} }
// Backfill performs a backfill request to the given server. // Backfill performs a backfill request to the given server.

View File

@ -205,6 +205,7 @@ func (r *messagesReq) retrieveEvents() (
} }
var events []gomatrixserverlib.HeaderedEvent var events []gomatrixserverlib.HeaderedEvent
util.GetLogger(r.ctx).WithField("start", start).WithField("end", end).Infof("Fetched %d events locally", len(streamEvents))
// There can be two reasons for streamEvents to be empty: either we've // There can be two reasons for streamEvents to be empty: either we've
// reached the oldest event in the room (or the most recent one, depending // reached the oldest event in the room (or the most recent one, depending
@ -373,11 +374,11 @@ func (e eventsByDepth) Less(i, j int) bool {
// event, or if there is no remote homeserver to contact. // event, or if there is no remote homeserver to contact.
// Returns an error if there was an issue with retrieving the list of servers in // Returns an error if there was an issue with retrieving the list of servers in
// the room or sending the request. // the room or sending the request.
func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) {
var res api.QueryBackfillResponse var res api.QueryBackfillResponse
err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{
RoomID: roomID, RoomID: roomID,
EarliestEventsIDs: fromEventIDs, BackwardsExtremities: backwardsExtremities,
Limit: limit, Limit: limit,
ServerName: r.cfg.Matrix.ServerName, ServerName: r.cfg.Matrix.ServerName,
}, &res) }, &res)

View File

@ -94,9 +94,8 @@ type Database interface {
GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error)
// EventPositionInTopology returns the depth and stream position of the given event. // EventPositionInTopology returns the depth and stream position of the given event.
EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error) EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error)
// BackwardExtremitiesForRoom returns the event IDs of all of the backward // BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events.
// extremities we know of for a given room. BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities map[string][]string, err error)
BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error)
// MaxTopologicalPosition returns the highest topological position for a given room. // MaxTopologicalPosition returns the highest topological position for a given room.
MaxTopologicalPosition(ctx context.Context, roomID string) (types.TopologyToken, error) MaxTopologicalPosition(ctx context.Context, roomID string) (types.TopologyToken, error)
// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and

View File

@ -41,7 +41,7 @@ const insertBackwardExtremitySQL = "" +
" ON CONFLICT DO NOTHING" " ON CONFLICT DO NOTHING"
const selectBackwardExtremitiesForRoomSQL = "" + const selectBackwardExtremitiesForRoomSQL = "" +
"SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" "SELECT event_id, prev_event_id FROM syncapi_backward_extremities WHERE room_id = $1"
const deleteBackwardExtremitySQL = "" + const deleteBackwardExtremitySQL = "" +
"DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2"
@ -79,23 +79,24 @@ func (s *backwardExtremitiesStatements) InsertsBackwardExtremity(
func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom( func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom(
ctx context.Context, roomID string, ctx context.Context, roomID string,
) (eventIDs []string, err error) { ) (bwExtrems map[string][]string, err error) {
rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID)
if err != nil { if err != nil {
return return
} }
defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed")
bwExtrems = make(map[string][]string)
for rows.Next() { for rows.Next() {
var eID string var eID string
if err = rows.Scan(&eID); err != nil { var prevEventID string
if err = rows.Scan(&eID, &prevEventID); err != nil {
return return
} }
bwExtrems[eID] = append(bwExtrems[eID], prevEventID)
eventIDs = append(eventIDs, eID)
} }
return eventIDs, rows.Err() return bwExtrems, rows.Err()
} }
func (s *backwardExtremitiesStatements) DeleteBackwardExtremity( func (s *backwardExtremitiesStatements) DeleteBackwardExtremity(

View File

@ -310,6 +310,7 @@ func (d *Database) updateRoomState(
} }
membership = &value membership = &value
} }
if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil {
return err return err
} }
@ -367,7 +368,7 @@ func (d *Database) SyncPosition(ctx context.Context) (tok types.StreamingToken,
func (d *Database) BackwardExtremitiesForRoom( func (d *Database) BackwardExtremitiesForRoom(
ctx context.Context, roomID string, ctx context.Context, roomID string,
) (backwardExtremities []string, err error) { ) (backwardExtremities map[string][]string, err error) {
return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID)
} }

View File

@ -41,7 +41,7 @@ const insertBackwardExtremitySQL = "" +
" ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING"
const selectBackwardExtremitiesForRoomSQL = "" + const selectBackwardExtremitiesForRoomSQL = "" +
"SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" "SELECT event_id, prev_event_id FROM syncapi_backward_extremities WHERE room_id = $1"
const deleteBackwardExtremitySQL = "" + const deleteBackwardExtremitySQL = "" +
"DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2"
@ -79,23 +79,24 @@ func (s *backwardExtremitiesStatements) InsertsBackwardExtremity(
func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom( func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom(
ctx context.Context, roomID string, ctx context.Context, roomID string,
) (eventIDs []string, err error) { ) (bwExtrems map[string][]string, err error) {
rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID)
if err != nil { if err != nil {
return return
} }
defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed")
bwExtrems = make(map[string][]string)
for rows.Next() { for rows.Next() {
var eID string var eID string
if err = rows.Scan(&eID); err != nil { var prevEventID string
if err = rows.Scan(&eID, &prevEventID); err != nil {
return return
} }
bwExtrems[eID] = append(bwExtrems[eID], prevEventID)
eventIDs = append(eventIDs, eID)
} }
return eventIDs, rows.Err() return bwExtrems, rows.Err()
} }
func (s *backwardExtremitiesStatements) DeleteBackwardExtremity( func (s *backwardExtremitiesStatements) DeleteBackwardExtremity(

View File

@ -89,8 +89,8 @@ type CurrentRoomState interface {
type BackwardsExtremities interface { type BackwardsExtremities interface {
// InsertsBackwardExtremity inserts a new backwards extremity. // InsertsBackwardExtremity inserts a new backwards extremity.
InsertsBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string) (err error) InsertsBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string) (err error)
// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room. // SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room, as a map of event_id to list of prev_event_ids.
SelectBackwardExtremitiesForRoom(ctx context.Context, roomID string) (eventIDs []string, err error) SelectBackwardExtremitiesForRoom(ctx context.Context, roomID string) (bwExtrems map[string][]string, err error)
// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. // DeleteBackwardExtremity removes a backwards extremity for a room, if one existed.
DeleteBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, knownEventID string) (err error) DeleteBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, knownEventID string) (err error)
} }

View File

@ -280,3 +280,4 @@ Inbound federation can get public room list
An event which redacts itself should be ignored An event which redacts itself should be ignored
A pair of events which redact each other should be ignored A pair of events which redact each other should be ignored
Outbound federation can backfill events Outbound federation can backfill events
Inbound federation can backfill events