Optimise servers to backfill from (#1485)

- Prefer perspective servers if they are in the room.
- Limit the number of backfill servers to 5 to avoid taking too long.
This commit is contained in:
Kegsay 2020-10-06 18:09:02 +01:00 committed by GitHub
parent 4feff8e8d9
commit 0f7e707f39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 58 additions and 22 deletions

View File

@ -26,28 +26,30 @@ type RoomserverInternalAPI struct {
*perform.Leaver *perform.Leaver
*perform.Publisher *perform.Publisher
*perform.Backfiller *perform.Backfiller
DB storage.Database DB storage.Database
Cfg *config.RoomServer Cfg *config.RoomServer
Producer sarama.SyncProducer Producer sarama.SyncProducer
Cache caching.RoomServerCaches Cache caching.RoomServerCaches
ServerName gomatrixserverlib.ServerName ServerName gomatrixserverlib.ServerName
KeyRing gomatrixserverlib.JSONVerifier KeyRing gomatrixserverlib.JSONVerifier
fsAPI fsAPI.FederationSenderInternalAPI fsAPI fsAPI.FederationSenderInternalAPI
OutputRoomEventTopic string // Kafka topic for new output room events OutputRoomEventTopic string // Kafka topic for new output room events
PerspectiveServerNames []gomatrixserverlib.ServerName
} }
func NewRoomserverAPI( func NewRoomserverAPI(
cfg *config.RoomServer, roomserverDB storage.Database, producer sarama.SyncProducer, cfg *config.RoomServer, roomserverDB storage.Database, producer sarama.SyncProducer,
outputRoomEventTopic string, caches caching.RoomServerCaches, outputRoomEventTopic string, caches caching.RoomServerCaches,
keyRing gomatrixserverlib.JSONVerifier, keyRing gomatrixserverlib.JSONVerifier, perspectiveServerNames []gomatrixserverlib.ServerName,
) *RoomserverInternalAPI { ) *RoomserverInternalAPI {
serverACLs := acls.NewServerACLs(roomserverDB) serverACLs := acls.NewServerACLs(roomserverDB)
a := &RoomserverInternalAPI{ a := &RoomserverInternalAPI{
DB: roomserverDB, DB: roomserverDB,
Cfg: cfg, Cfg: cfg,
Cache: caches, Cache: caches,
ServerName: cfg.Matrix.ServerName, ServerName: cfg.Matrix.ServerName,
KeyRing: keyRing, PerspectiveServerNames: perspectiveServerNames,
KeyRing: keyRing,
Queryer: &query.Queryer{ Queryer: &query.Queryer{
DB: roomserverDB, DB: roomserverDB,
Cache: caches, Cache: caches,
@ -105,6 +107,10 @@ func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSen
DB: r.DB, DB: r.DB,
FSAPI: r.fsAPI, FSAPI: r.fsAPI,
KeyRing: r.KeyRing, KeyRing: r.KeyRing,
// Perspective servers are trusted to not lie about server keys, so we will also
// prefer these servers when backfilling (assuming they are in the room) rather
// than trying random servers
PreferServers: r.PerspectiveServerNames,
} }
} }

View File

@ -30,11 +30,19 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
// the max number of servers to backfill from per request. If this is too low we may fail to backfill when
// we could've from another server. If this is too high we may take far too long to successfully backfill
// as we try dead servers.
const maxBackfillServers = 5
type Backfiller struct { type Backfiller struct {
ServerName gomatrixserverlib.ServerName ServerName gomatrixserverlib.ServerName
DB storage.Database DB storage.Database
FSAPI federationSenderAPI.FederationSenderInternalAPI FSAPI federationSenderAPI.FederationSenderInternalAPI
KeyRing gomatrixserverlib.JSONVerifier KeyRing gomatrixserverlib.JSONVerifier
// The servers which should be preferred above other servers when backfilling
PreferServers []gomatrixserverlib.ServerName
} }
// PerformBackfill implements api.RoomServerQueryAPI // PerformBackfill implements api.RoomServerQueryAPI
@ -96,7 +104,7 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform
if info == nil || info.IsStub { if info == nil || info.IsStub {
return fmt.Errorf("backfillViaFederation: missing room info for room %s", req.RoomID) return fmt.Errorf("backfillViaFederation: missing room info for room %s", req.RoomID)
} }
requester := newBackfillRequester(r.DB, r.FSAPI, r.ServerName, req.BackwardsExtremities) requester := newBackfillRequester(r.DB, r.FSAPI, r.ServerName, req.BackwardsExtremities, r.PreferServers)
// Request 100 items regardless of what the query asks for. // Request 100 items regardless of what the query asks for.
// We don't want to go much higher than this. // We don't want to go much higher than this.
// We can't honour exactly the limit as some sytests rely on requesting more for tests to pass // We can't honour exactly the limit as some sytests rely on requesting more for tests to pass
@ -215,10 +223,11 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom
// backfillRequester implements gomatrixserverlib.BackfillRequester // backfillRequester implements gomatrixserverlib.BackfillRequester
type backfillRequester struct { type backfillRequester struct {
db storage.Database db storage.Database
fsAPI federationSenderAPI.FederationSenderInternalAPI fsAPI federationSenderAPI.FederationSenderInternalAPI
thisServer gomatrixserverlib.ServerName thisServer gomatrixserverlib.ServerName
bwExtrems map[string][]string preferServer map[gomatrixserverlib.ServerName]bool
bwExtrems map[string][]string
// per-request state // per-request state
servers []gomatrixserverlib.ServerName servers []gomatrixserverlib.ServerName
@ -226,7 +235,14 @@ type backfillRequester struct {
eventIDMap map[string]gomatrixserverlib.Event eventIDMap map[string]gomatrixserverlib.Event
} }
func newBackfillRequester(db storage.Database, fsAPI federationSenderAPI.FederationSenderInternalAPI, thisServer gomatrixserverlib.ServerName, bwExtrems map[string][]string) *backfillRequester { func newBackfillRequester(
db storage.Database, fsAPI federationSenderAPI.FederationSenderInternalAPI, thisServer gomatrixserverlib.ServerName,
bwExtrems map[string][]string, preferServers []gomatrixserverlib.ServerName,
) *backfillRequester {
preferServer := make(map[gomatrixserverlib.ServerName]bool)
for _, p := range preferServers {
preferServer[p] = true
}
return &backfillRequester{ return &backfillRequester{
db: db, db: db,
fsAPI: fsAPI, fsAPI: fsAPI,
@ -234,6 +250,7 @@ func newBackfillRequester(db storage.Database, fsAPI federationSenderAPI.Federat
eventIDToBeforeStateIDs: make(map[string][]string), eventIDToBeforeStateIDs: make(map[string][]string),
eventIDMap: make(map[string]gomatrixserverlib.Event), eventIDMap: make(map[string]gomatrixserverlib.Event),
bwExtrems: bwExtrems, bwExtrems: bwExtrems,
preferServer: preferServer,
} }
} }
@ -436,8 +453,16 @@ FindSuccessor:
if server == b.thisServer { if server == b.thisServer {
continue continue
} }
servers = append(servers, server) if b.preferServer[server] { // insert at the front
servers = append([]gomatrixserverlib.ServerName{server}, servers...)
} else { // insert at the back
servers = append(servers, server)
}
} }
if len(servers) > maxBackfillServers {
servers = servers[:maxBackfillServers]
}
b.servers = servers b.servers = servers
return servers return servers
} }

View File

@ -41,6 +41,11 @@ func NewInternalAPI(
) api.RoomserverInternalAPI { ) api.RoomserverInternalAPI {
cfg := &base.Cfg.RoomServer cfg := &base.Cfg.RoomServer
var perspectiveServerNames []gomatrixserverlib.ServerName
for _, kp := range base.Cfg.ServerKeyAPI.KeyPerspectives {
perspectiveServerNames = append(perspectiveServerNames, kp.ServerName)
}
roomserverDB, err := storage.Open(&cfg.Database, base.Caches) roomserverDB, err := storage.Open(&cfg.Database, base.Caches)
if err != nil { if err != nil {
logrus.WithError(err).Panicf("failed to connect to room server db") logrus.WithError(err).Panicf("failed to connect to room server db")
@ -48,6 +53,6 @@ func NewInternalAPI(
return internal.NewRoomserverAPI( return internal.NewRoomserverAPI(
cfg, roomserverDB, base.KafkaProducer, string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputRoomEvent)), cfg, roomserverDB, base.KafkaProducer, string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputRoomEvent)),
base.Caches, keyRing, base.Caches, keyRing, perspectiveServerNames,
) )
} }