Send-to-device messages over federation (#1198)

* Initial work to send send-to-device messages over federation

* Wire up send-to-device consumer, message formatting

* Generate random message ID

* Review comments, update sytest whitelist
This commit is contained in:
Neil Alexander 2020-07-14 12:33:37 +01:00 committed by GitHub
parent 5355c289b9
commit 72b3160776
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 99 additions and 31 deletions

View File

@ -15,6 +15,7 @@ package consumers
import (
"context"
"encoding/json"
"fmt"
"github.com/Shopify/sarama"
"github.com/matrix-org/dendrite/eduserver/api"
@ -23,53 +24,120 @@ import (
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
log "github.com/sirupsen/logrus"
)
// OutputTypingEventConsumer consumes events that originate in EDU server.
type OutputTypingEventConsumer struct {
consumer *internal.ContinualConsumer
db storage.Database
queues *queue.OutgoingQueues
ServerName gomatrixserverlib.ServerName
// OutputEDUConsumer consumes events that originate in EDU server.
type OutputEDUConsumer struct {
typingConsumer *internal.ContinualConsumer
sendToDeviceConsumer *internal.ContinualConsumer
db storage.Database
queues *queue.OutgoingQueues
ServerName gomatrixserverlib.ServerName
TypingTopic string
SendToDeviceTopic string
}
// NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. Call Start() to begin consuming from EDU servers.
func NewOutputTypingEventConsumer(
// NewOutputEDUConsumer creates a new OutputEDUConsumer. Call Start() to begin consuming from EDU servers.
func NewOutputEDUConsumer(
cfg *config.Dendrite,
kafkaConsumer sarama.Consumer,
queues *queue.OutgoingQueues,
store storage.Database,
) *OutputTypingEventConsumer {
consumer := internal.ContinualConsumer{
Topic: string(cfg.Kafka.Topics.OutputTypingEvent),
Consumer: kafkaConsumer,
PartitionStore: store,
) *OutputEDUConsumer {
c := &OutputEDUConsumer{
typingConsumer: &internal.ContinualConsumer{
Topic: string(cfg.Kafka.Topics.OutputTypingEvent),
Consumer: kafkaConsumer,
PartitionStore: store,
},
sendToDeviceConsumer: &internal.ContinualConsumer{
Topic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent),
Consumer: kafkaConsumer,
PartitionStore: store,
},
queues: queues,
db: store,
ServerName: cfg.Matrix.ServerName,
TypingTopic: string(cfg.Kafka.Topics.OutputTypingEvent),
SendToDeviceTopic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent),
}
c := &OutputTypingEventConsumer{
consumer: &consumer,
queues: queues,
db: store,
ServerName: cfg.Matrix.ServerName,
}
consumer.ProcessMessage = c.onMessage
c.typingConsumer.ProcessMessage = c.onTypingEvent
c.sendToDeviceConsumer.ProcessMessage = c.onSendToDeviceEvent
return c
}
// Start consuming from EDU servers
func (t *OutputTypingEventConsumer) Start() error {
return t.consumer.Start()
func (t *OutputEDUConsumer) Start() error {
if err := t.typingConsumer.Start(); err != nil {
return fmt.Errorf("t.typingConsumer.Start: %w", err)
}
if err := t.sendToDeviceConsumer.Start(); err != nil {
return fmt.Errorf("t.sendToDeviceConsumer.Start: %w", err)
}
return nil
}
// onMessage is called for OutputTypingEvent received from the EDU servers.
// Parses the msg, creates a matrix federation EDU and sends it to joined hosts.
func (t *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
// onSendToDeviceEvent is called in response to a message received on the
// send-to-device events topic from the EDU server.
func (t *OutputEDUConsumer) onSendToDeviceEvent(msg *sarama.ConsumerMessage) error {
// Extract the send-to-device event from msg.
var ote api.OutputSendToDeviceEvent
if err := json.Unmarshal(msg.Value, &ote); err != nil {
log.WithError(err).Errorf("eduserver output log: message parse failed (expected send-to-device)")
return nil
}
// only send send-to-device events which originated from us
_, originServerName, err := gomatrixserverlib.SplitID('@', ote.Sender)
if err != nil {
log.WithError(err).WithField("user_id", ote.Sender).Error("Failed to extract domain from send-to-device sender")
return nil
}
if originServerName != t.ServerName {
log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere")
return nil
}
_, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID)
if err != nil {
log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination")
return nil
}
// Pack the EDU and marshal it
edu := &gomatrixserverlib.EDU{
Type: gomatrixserverlib.MDirectToDevice,
Origin: string(t.ServerName),
}
tdm := gomatrixserverlib.ToDeviceMessage{
Sender: ote.Sender,
Type: ote.Type,
MessageID: util.RandomString(32),
Messages: map[string]map[string]json.RawMessage{
ote.UserID: {
ote.DeviceID: ote.Content,
},
},
}
if edu.Content, err = json.Marshal(tdm); err != nil {
return err
}
log.Infof("Sending send-to-device message into %q destination queue", destServerName)
return t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName})
}
// onTypingEvent is called in response to a message received on the typing
// events topic from the EDU server.
func (t *OutputEDUConsumer) onTypingEvent(msg *sarama.ConsumerMessage) error {
// Extract the typing event from msg.
var ote api.OutputTypingEvent
if err := json.Unmarshal(msg.Value, &ote); err != nil {
// Skip this msg but continue processing messages.
log.WithError(err).Errorf("eduserver output log: message parse failed")
log.WithError(err).Errorf("eduserver output log: message parse failed (expected typing)")
return nil
}

View File

@ -66,7 +66,7 @@ func NewInternalAPI(
logrus.WithError(err).Panic("failed to start room server consumer")
}
tsConsumer := consumers.NewOutputTypingEventConsumer(
tsConsumer := consumers.NewOutputEDUConsumer(
base.Cfg, base.KafkaConsumer, queues, federationSenderDB,
)
if err := tsConsumer.Start(); err != nil {

View File

@ -174,7 +174,7 @@ func (oqs *OutgoingQueues) SendInvite(
return nil
}
// SendEDU sends an EDU event to the destinations
// SendEDU sends an EDU event to the destinations.
func (oqs *OutgoingQueues) SendEDU(
e *gomatrixserverlib.EDU, origin gomatrixserverlib.ServerName,
destinations []gomatrixserverlib.ServerName,

View File

@ -316,12 +316,12 @@ Can recv a device message using /sync
Can recv device messages until they are acknowledged
Device messages with the same txn_id are deduplicated
Device messages wake up /sync
# TODO: separate PR for: Can recv device messages over federation
# TODO: separate PR for: Device messages over federation wake up /sync
Can recv device messages over federation
Device messages over federation wake up /sync
Can send messages with a wildcard device id
Can send messages with a wildcard device id to two devices
Wildcard device messages wake up /sync
# TODO: separate PR for: Wildcard device messages over federation wake up /sync
Wildcard device messages over federation wake up /sync
Can send a to-device message to two users which both receive it using /sync
User can create and send/receive messages in a room with version 6
local user can join room with version 6