p2p: de-dupe and shuffle public rooms; implement keydb (#1074)

- We don't want a serverKeyAPI as fetching keys doesn't need a DB.
- De-dupe rooms so we don't see them multiple times, but shuffle the
  alias we join via so we don't all flood a single server.
This commit is contained in:
Kegsay 2020-05-29 15:34:15 +01:00 committed by GitHub
parent fe5cf6f880
commit 97c64bdb6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 26 additions and 6 deletions

View File

@ -82,3 +82,8 @@ func (f *libp2pKeyFetcher) FetchKeys(
func (f *libp2pKeyFetcher) FetcherName() string {
return "libp2pKeyFetcher"
}
// no-op function for storing keys - we don't do any work to fetch them so don't bother storing.
func (f *libp2pKeyFetcher) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error {
return nil
}

View File

@ -37,7 +37,6 @@ import (
"github.com/matrix-org/dendrite/publicroomsapi"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/serverkeyapi"
"github.com/matrix-org/dendrite/syncapi"
go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
"github.com/matrix-org/gomatrixserverlib"
@ -197,14 +196,12 @@ func main() {
deviceDB := base.CreateDeviceDB()
federation := createFederationClient(cfg, node)
serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent(
base, federation,
)
fetcher := &libp2pKeyFetcher{}
keyRing := gomatrixserverlib.KeyRing{
KeyFetchers: []gomatrixserverlib.KeyFetcher{
&libp2pKeyFetcher{},
fetcher,
},
KeyDatabase: serverKeyAPI,
KeyDatabase: fetcher,
}
p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node)

View File

@ -16,6 +16,7 @@ package directory
import (
"context"
"math/rand"
"net/http"
"strconv"
"sync"
@ -96,6 +97,23 @@ func GetPostPublicRoomsWithExternal(
// downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16
fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit))
response.Chunk = append(response.Chunk, fedRooms...)
// de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers
// are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit)
var publicRooms []gomatrixserverlib.PublicRoom
haveRoomIDs := make(map[string]bool)
rand.Shuffle(len(response.Chunk), func(i, j int) {
response.Chunk[i], response.Chunk[j] = response.Chunk[j], response.Chunk[i]
})
for _, r := range response.Chunk {
if haveRoomIDs[r.RoomID] {
continue
}
haveRoomIDs[r.RoomID] = true
publicRooms = append(publicRooms, r)
}
response.Chunk = publicRooms
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,