2017-06-28 16:10:17 +01:00
|
|
|
// Copyright 2017 Vector Creations Ltd
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package queue
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"sync"
|
|
|
|
|
2020-04-28 10:53:07 +01:00
|
|
|
"github.com/matrix-org/dendrite/federationsender/producers"
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
"github.com/matrix-org/dendrite/federationsender/types"
|
2017-06-28 16:10:17 +01:00
|
|
|
"github.com/matrix-org/gomatrixserverlib"
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
"github.com/matrix-org/util"
|
2017-11-16 10:12:02 +00:00
|
|
|
log "github.com/sirupsen/logrus"
|
2017-06-28 16:10:17 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
// OutgoingQueues is a collection of queues for sending transactions to other
|
|
|
|
// matrix servers
|
|
|
|
type OutgoingQueues struct {
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
rsProducer *producers.RoomserverProducer
|
|
|
|
origin gomatrixserverlib.ServerName
|
|
|
|
client *gomatrixserverlib.FederationClient
|
|
|
|
statistics *types.Statistics
|
|
|
|
queuesMutex sync.Mutex // protects the below
|
2017-06-28 16:10:17 +01:00
|
|
|
queues map[gomatrixserverlib.ServerName]*destinationQueue
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewOutgoingQueues makes a new OutgoingQueues
|
2020-04-28 10:53:07 +01:00
|
|
|
func NewOutgoingQueues(
|
|
|
|
origin gomatrixserverlib.ServerName,
|
|
|
|
client *gomatrixserverlib.FederationClient,
|
|
|
|
rsProducer *producers.RoomserverProducer,
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
statistics *types.Statistics,
|
2020-04-28 10:53:07 +01:00
|
|
|
) *OutgoingQueues {
|
2017-06-28 16:10:17 +01:00
|
|
|
return &OutgoingQueues{
|
2020-04-28 10:53:07 +01:00
|
|
|
rsProducer: rsProducer,
|
|
|
|
origin: origin,
|
|
|
|
client: client,
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
statistics: statistics,
|
2020-04-28 10:53:07 +01:00
|
|
|
queues: map[gomatrixserverlib.ServerName]*destinationQueue{},
|
2017-06-28 16:10:17 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue {
|
|
|
|
oqs.queuesMutex.Lock()
|
|
|
|
defer oqs.queuesMutex.Unlock()
|
|
|
|
oq := oqs.queues[destination]
|
|
|
|
if oq == nil {
|
|
|
|
oq = &destinationQueue{
|
|
|
|
rsProducer: oqs.rsProducer,
|
|
|
|
origin: oqs.origin,
|
|
|
|
destination: destination,
|
|
|
|
client: oqs.client,
|
|
|
|
statistics: oqs.statistics.ForServer(destination),
|
|
|
|
incomingPDUs: make(chan *gomatrixserverlib.HeaderedEvent, 128),
|
|
|
|
incomingEDUs: make(chan *gomatrixserverlib.EDU, 128),
|
|
|
|
incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128),
|
|
|
|
}
|
|
|
|
oqs.queues[destination] = oq
|
|
|
|
}
|
|
|
|
return oq
|
|
|
|
}
|
|
|
|
|
2017-06-28 16:10:17 +01:00
|
|
|
// SendEvent sends an event to the destinations
|
|
|
|
func (oqs *OutgoingQueues) SendEvent(
|
2020-03-27 16:28:22 +00:00
|
|
|
ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName,
|
2017-06-28 16:10:17 +01:00
|
|
|
destinations []gomatrixserverlib.ServerName,
|
|
|
|
) error {
|
|
|
|
if origin != oqs.origin {
|
2018-08-10 16:26:57 +01:00
|
|
|
// TODO: Support virtual hosting; gh issue #577.
|
2017-06-28 16:10:17 +01:00
|
|
|
return fmt.Errorf(
|
|
|
|
"sendevent: unexpected server to send as: got %q expected %q",
|
|
|
|
origin, oqs.origin,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove our own server from the list of destinations.
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
destinations = filterAndDedupeDests(oqs.origin, destinations)
|
2017-06-28 16:10:17 +01:00
|
|
|
|
|
|
|
log.WithFields(log.Fields{
|
|
|
|
"destinations": destinations, "event": ev.EventID(),
|
|
|
|
}).Info("Sending event")
|
|
|
|
|
|
|
|
for _, destination := range destinations {
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
oqs.getQueue(destination).sendEvent(ev)
|
2017-06-28 16:10:17 +01:00
|
|
|
}
|
2018-08-10 16:26:57 +01:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-04-03 14:29:06 +01:00
|
|
|
// SendEvent sends an event to the destinations
|
|
|
|
func (oqs *OutgoingQueues) SendInvite(
|
|
|
|
inviteReq *gomatrixserverlib.InviteV2Request,
|
|
|
|
) error {
|
|
|
|
ev := inviteReq.Event()
|
|
|
|
stateKey := ev.StateKey()
|
|
|
|
if stateKey == nil {
|
|
|
|
log.WithFields(log.Fields{
|
|
|
|
"event_id": ev.EventID(),
|
|
|
|
}).Info("invite had no state key, dropping")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
_, destination, err := gomatrixserverlib.SplitID('@', *stateKey)
|
|
|
|
if err != nil {
|
|
|
|
log.WithFields(log.Fields{
|
|
|
|
"event_id": ev.EventID(),
|
|
|
|
"state_key": stateKey,
|
|
|
|
}).Info("failed to split destination from state key")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
log.WithFields(log.Fields{
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
"event_id": ev.EventID(),
|
|
|
|
"server_name": destination,
|
2020-04-03 14:29:06 +01:00
|
|
|
}).Info("Sending invite")
|
|
|
|
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
oqs.getQueue(destination).sendInvite(inviteReq)
|
2020-04-03 14:29:06 +01:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-08-10 16:26:57 +01:00
|
|
|
// SendEDU sends an EDU event to the destinations
|
|
|
|
func (oqs *OutgoingQueues) SendEDU(
|
|
|
|
e *gomatrixserverlib.EDU, origin gomatrixserverlib.ServerName,
|
|
|
|
destinations []gomatrixserverlib.ServerName,
|
|
|
|
) error {
|
|
|
|
if origin != oqs.origin {
|
|
|
|
// TODO: Support virtual hosting; gh issue #577.
|
|
|
|
return fmt.Errorf(
|
|
|
|
"sendevent: unexpected server to send as: got %q expected %q",
|
|
|
|
origin, oqs.origin,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove our own server from the list of destinations.
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
destinations = filterAndDedupeDests(oqs.origin, destinations)
|
2018-08-10 16:26:57 +01:00
|
|
|
|
2019-07-01 16:04:49 +01:00
|
|
|
if len(destinations) > 0 {
|
|
|
|
log.WithFields(log.Fields{
|
|
|
|
"destinations": destinations, "edu_type": e.Type,
|
|
|
|
}).Info("Sending EDU event")
|
|
|
|
}
|
2018-08-10 16:26:57 +01:00
|
|
|
|
|
|
|
for _, destination := range destinations {
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
oqs.getQueue(destination).sendEDU(e)
|
2018-08-10 16:26:57 +01:00
|
|
|
}
|
|
|
|
|
2017-06-28 16:10:17 +01:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
// filterAndDedupeDests removes our own server from the list of destinations
|
|
|
|
// and deduplicates any servers in the list that may appear more than once.
|
|
|
|
func filterAndDedupeDests(origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName) (
|
|
|
|
result []gomatrixserverlib.ServerName,
|
|
|
|
) {
|
|
|
|
strs := make([]string, len(destinations))
|
|
|
|
for i, d := range destinations {
|
|
|
|
strs[i] = string(d)
|
|
|
|
}
|
|
|
|
for _, destination := range util.UniqueStrings(strs) {
|
|
|
|
if gomatrixserverlib.ServerName(destination) == origin {
|
2017-06-28 16:10:17 +01:00
|
|
|
continue
|
|
|
|
}
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 12:42:06 +01:00
|
|
|
result = append(result, gomatrixserverlib.ServerName(destination))
|
2017-06-28 16:10:17 +01:00
|
|
|
}
|
|
|
|
return result
|
|
|
|
}
|