NATS JetStream tweaks (#2086)

* Use named NATS durable consumers

* Build fixes

* Remove dupe call to SetFederationAPI

* Use namespaced consumer name

* Fix namespacing

* Fix unit tests hopefully
This commit is contained in:
Neil Alexander 2022-01-07 17:31:57 +00:00 committed by GitHub
parent a422321435
commit 16035b9737
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 51 additions and 15 deletions

View File

@ -34,6 +34,7 @@ import (
type OutputRoomEventConsumer struct { type OutputRoomEventConsumer struct {
ctx context.Context ctx context.Context
jetstream nats.JetStreamContext jetstream nats.JetStreamContext
durable nats.SubOpt
topic string topic string
asDB storage.Database asDB storage.Database
rsAPI api.RoomserverInternalAPI rsAPI api.RoomserverInternalAPI
@ -54,6 +55,7 @@ func NewOutputRoomEventConsumer(
return &OutputRoomEventConsumer{ return &OutputRoomEventConsumer{
ctx: process.Context(), ctx: process.Context(),
jetstream: js, jetstream: js,
durable: cfg.Global.JetStream.Durable("AppserviceRoomserverConsumer"),
topic: cfg.Global.JetStream.TopicFor(jetstream.OutputRoomEvent), topic: cfg.Global.JetStream.TopicFor(jetstream.OutputRoomEvent),
asDB: appserviceDB, asDB: appserviceDB,
rsAPI: rsAPI, rsAPI: rsAPI,
@ -64,7 +66,7 @@ func NewOutputRoomEventConsumer(
// Start consuming from room servers // Start consuming from room servers
func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) Start() error {
_, err := s.jetstream.Subscribe(s.topic, s.onMessage) _, err := s.jetstream.Subscribe(s.topic, s.onMessage, s.durable)
return err return err
} }

View File

@ -99,10 +99,6 @@ func main() {
} }
keyRing := fsAPI.KeyRing() keyRing := fsAPI.KeyRing()
// The underlying roomserver implementation needs to be able to call the fedsender.
// This is different to rsAPI which can be the http client which doesn't need this dependency
rsImpl.SetFederationAPI(fsAPI, keyRing)
keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
keyAPI := keyImpl keyAPI := keyImpl
if base.UseHTTPAPIs { if base.UseHTTPAPIs {

View File

@ -34,6 +34,7 @@ import (
type OutputEDUConsumer struct { type OutputEDUConsumer struct {
ctx context.Context ctx context.Context
jetstream nats.JetStreamContext jetstream nats.JetStreamContext
durable nats.SubOpt
db storage.Database db storage.Database
queues *queue.OutgoingQueues queues *queue.OutgoingQueues
ServerName gomatrixserverlib.ServerName ServerName gomatrixserverlib.ServerName
@ -56,6 +57,7 @@ func NewOutputEDUConsumer(
queues: queues, queues: queues,
db: store, db: store,
ServerName: cfg.Matrix.ServerName, ServerName: cfg.Matrix.ServerName,
durable: cfg.Matrix.JetStream.Durable("FederationAPIEDUServerConsumer"),
typingTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputTypingEvent), typingTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputTypingEvent),
sendToDeviceTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputSendToDeviceEvent), sendToDeviceTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputSendToDeviceEvent),
receiptTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReceiptEvent), receiptTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReceiptEvent),
@ -64,13 +66,13 @@ func NewOutputEDUConsumer(
// Start consuming from EDU servers // Start consuming from EDU servers
func (t *OutputEDUConsumer) Start() error { func (t *OutputEDUConsumer) Start() error {
if _, err := t.jetstream.Subscribe(t.typingTopic, t.onTypingEvent); err != nil { if _, err := t.jetstream.Subscribe(t.typingTopic, t.onTypingEvent, t.durable); err != nil {
return err return err
} }
if _, err := t.jetstream.Subscribe(t.sendToDeviceTopic, t.onSendToDeviceEvent); err != nil { if _, err := t.jetstream.Subscribe(t.sendToDeviceTopic, t.onSendToDeviceEvent, t.durable); err != nil {
return err return err
} }
if _, err := t.jetstream.Subscribe(t.receiptTopic, t.onReceiptEvent); err != nil { if _, err := t.jetstream.Subscribe(t.receiptTopic, t.onReceiptEvent, t.durable); err != nil {
return err return err
} }
return nil return nil

View File

@ -37,6 +37,7 @@ type OutputRoomEventConsumer struct {
cfg *config.FederationAPI cfg *config.FederationAPI
rsAPI api.RoomserverInternalAPI rsAPI api.RoomserverInternalAPI
jetstream nats.JetStreamContext jetstream nats.JetStreamContext
durable nats.SubOpt
db storage.Database db storage.Database
queues *queue.OutgoingQueues queues *queue.OutgoingQueues
topic string topic string
@ -58,13 +59,14 @@ func NewOutputRoomEventConsumer(
db: store, db: store,
queues: queues, queues: queues,
rsAPI: rsAPI, rsAPI: rsAPI,
durable: cfg.Matrix.JetStream.Durable("FederationAPIRoomServerConsumer"),
topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputRoomEvent), topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputRoomEvent),
} }
} }
// Start consuming from room servers // Start consuming from room servers
func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) Start() error {
_, err := s.jetstream.Subscribe(s.topic, s.onMessage) _, err := s.jetstream.Subscribe(s.topic, s.onMessage, s.durable)
return err return err
} }

View File

@ -68,6 +68,13 @@ func TestMain(m *testing.M) {
panic("can't create cache: " + err.Error()) panic("can't create cache: " + err.Error())
} }
// Create a temporary directory for JetStream.
d, err := ioutil.TempDir("./", "jetstream*")
if err != nil {
panic(err)
}
defer os.RemoveAll(d)
// Draw up just enough Dendrite config for the server key // Draw up just enough Dendrite config for the server key
// API to work. // API to work.
cfg := &config.Dendrite{} cfg := &config.Dendrite{}
@ -75,6 +82,8 @@ func TestMain(m *testing.M) {
cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name) cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
cfg.Global.PrivateKey = testPriv cfg.Global.PrivateKey = testPriv
cfg.Global.JetStream.InMemory = true cfg.Global.JetStream.InMemory = true
cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
cfg.Global.JetStream.StoragePath = config.Path(d)
cfg.Global.KeyID = serverKeyID cfg.Global.KeyID = serverKeyID
cfg.Global.KeyValidityPeriod = s.validity cfg.Global.KeyValidityPeriod = s.validity
cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:") cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")

View File

@ -67,6 +67,7 @@ func NewRoomserverAPI(
InputRoomEventTopic: inputRoomEventTopic, InputRoomEventTopic: inputRoomEventTopic,
OutputRoomEventTopic: outputRoomEventTopic, OutputRoomEventTopic: outputRoomEventTopic,
JetStream: consumer, JetStream: consumer,
Durable: cfg.Matrix.JetStream.Durable("RoomserverInputConsumer"),
ServerName: cfg.Matrix.ServerName, ServerName: cfg.Matrix.ServerName,
ACLs: serverACLs, ACLs: serverACLs,
}, },

View File

@ -43,6 +43,7 @@ var keyContentFields = map[string]string{
type Inputer struct { type Inputer struct {
DB storage.Database DB storage.Database
JetStream nats.JetStreamContext JetStream nats.JetStreamContext
Durable nats.SubOpt
ServerName gomatrixserverlib.ServerName ServerName gomatrixserverlib.ServerName
ACLs *acls.ServerACLs ACLs *acls.ServerACLs
InputRoomEventTopic string InputRoomEventTopic string
@ -85,6 +86,8 @@ func (r *Inputer) Start() error {
// or nak them within a certain amount of time. This stops that from // or nak them within a certain amount of time. This stops that from
// happening, so we don't end up doing a lot of unnecessary duplicate work. // happening, so we don't end up doing a lot of unnecessary duplicate work.
nats.MaxDeliver(0), nats.MaxDeliver(0),
// Use a durable named consumer.
r.Durable,
) )
return err return err
} }

View File

@ -2,6 +2,8 @@ package config
import ( import (
"fmt" "fmt"
"github.com/nats-io/nats.go"
) )
type JetStream struct { type JetStream struct {
@ -23,6 +25,10 @@ func (c *JetStream) TopicFor(name string) string {
return fmt.Sprintf("%s%s", c.TopicPrefix, name) return fmt.Sprintf("%s%s", c.TopicPrefix, name)
} }
func (c *JetStream) Durable(name string) nats.SubOpt {
return nats.Durable(c.TopicFor(name))
}
func (c *JetStream) Defaults(generate bool) { func (c *JetStream) Defaults(generate bool) {
c.Addresses = []string{} c.Addresses = []string{}
c.TopicPrefix = "Dendrite" c.TopicPrefix = "Dendrite"

View File

@ -75,13 +75,18 @@ func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (nats.JetStreamContex
} }
if info == nil { if info == nil {
stream.Subjects = []string{name} stream.Subjects = []string{name}
// If we're trying to keep everything in memory (e.g. unit tests) // If we're trying to keep everything in memory (e.g. unit tests)
// then overwrite the storage policy. // then overwrite the storage policy.
if cfg.InMemory { if cfg.InMemory {
stream.Storage = nats.MemoryStorage stream.Storage = nats.MemoryStorage
} }
if _, err = s.AddStream(stream); err != nil { // Namespace the streams without modifying the original streams
// array, otherwise we end up with namespaces on namespaces.
namespaced := *stream
namespaced.Name = name
if _, err = s.AddStream(&namespaced); err != nil {
logrus.WithError(err).WithField("stream", name).Fatal("Unable to add stream") logrus.WithError(err).WithField("stream", name).Fatal("Unable to add stream")
} }
} }

View File

@ -34,6 +34,7 @@ import (
type OutputClientDataConsumer struct { type OutputClientDataConsumer struct {
ctx context.Context ctx context.Context
jetstream nats.JetStreamContext jetstream nats.JetStreamContext
durable nats.SubOpt
topic string topic string
db storage.Database db storage.Database
stream types.StreamProvider stream types.StreamProvider
@ -53,6 +54,7 @@ func NewOutputClientDataConsumer(
ctx: process.Context(), ctx: process.Context(),
jetstream: js, jetstream: js,
topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputClientData), topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputClientData),
durable: cfg.Matrix.JetStream.Durable("SyncAPIClientAPIConsumer"),
db: store, db: store,
notifier: notifier, notifier: notifier,
stream: stream, stream: stream,
@ -61,7 +63,7 @@ func NewOutputClientDataConsumer(
// Start consuming from room servers // Start consuming from room servers
func (s *OutputClientDataConsumer) Start() error { func (s *OutputClientDataConsumer) Start() error {
_, err := s.jetstream.Subscribe(s.topic, s.onMessage) _, err := s.jetstream.Subscribe(s.topic, s.onMessage, s.durable)
return err return err
} }

View File

@ -34,6 +34,7 @@ import (
type OutputReceiptEventConsumer struct { type OutputReceiptEventConsumer struct {
ctx context.Context ctx context.Context
jetstream nats.JetStreamContext jetstream nats.JetStreamContext
durable nats.SubOpt
topic string topic string
db storage.Database db storage.Database
stream types.StreamProvider stream types.StreamProvider
@ -54,6 +55,7 @@ func NewOutputReceiptEventConsumer(
ctx: process.Context(), ctx: process.Context(),
jetstream: js, jetstream: js,
topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReceiptEvent), topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputReceiptEvent),
durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerReceiptConsumer"),
db: store, db: store,
notifier: notifier, notifier: notifier,
stream: stream, stream: stream,
@ -62,7 +64,7 @@ func NewOutputReceiptEventConsumer(
// Start consuming from EDU api // Start consuming from EDU api
func (s *OutputReceiptEventConsumer) Start() error { func (s *OutputReceiptEventConsumer) Start() error {
_, err := s.jetstream.Subscribe(s.topic, s.onMessage) _, err := s.jetstream.Subscribe(s.topic, s.onMessage, s.durable)
return err return err
} }

View File

@ -36,6 +36,7 @@ import (
type OutputSendToDeviceEventConsumer struct { type OutputSendToDeviceEventConsumer struct {
ctx context.Context ctx context.Context
jetstream nats.JetStreamContext jetstream nats.JetStreamContext
durable nats.SubOpt
topic string topic string
db storage.Database db storage.Database
serverName gomatrixserverlib.ServerName // our server name serverName gomatrixserverlib.ServerName // our server name
@ -57,6 +58,7 @@ func NewOutputSendToDeviceEventConsumer(
ctx: process.Context(), ctx: process.Context(),
jetstream: js, jetstream: js,
topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputSendToDeviceEvent), topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputSendToDeviceEvent),
durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerSendToDeviceConsumer"),
db: store, db: store,
serverName: cfg.Matrix.ServerName, serverName: cfg.Matrix.ServerName,
notifier: notifier, notifier: notifier,
@ -66,7 +68,7 @@ func NewOutputSendToDeviceEventConsumer(
// Start consuming from EDU api // Start consuming from EDU api
func (s *OutputSendToDeviceEventConsumer) Start() error { func (s *OutputSendToDeviceEventConsumer) Start() error {
_, err := s.jetstream.Subscribe(s.topic, s.onMessage) _, err := s.jetstream.Subscribe(s.topic, s.onMessage, s.durable)
return err return err
} }

View File

@ -35,6 +35,7 @@ import (
type OutputTypingEventConsumer struct { type OutputTypingEventConsumer struct {
ctx context.Context ctx context.Context
jetstream nats.JetStreamContext jetstream nats.JetStreamContext
durable nats.SubOpt
topic string topic string
eduCache *cache.EDUCache eduCache *cache.EDUCache
stream types.StreamProvider stream types.StreamProvider
@ -56,6 +57,7 @@ func NewOutputTypingEventConsumer(
ctx: process.Context(), ctx: process.Context(),
jetstream: js, jetstream: js,
topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputTypingEvent), topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputTypingEvent),
durable: cfg.Matrix.JetStream.Durable("SyncAPIEDUServerTypingConsumer"),
eduCache: eduCache, eduCache: eduCache,
notifier: notifier, notifier: notifier,
stream: stream, stream: stream,
@ -64,7 +66,7 @@ func NewOutputTypingEventConsumer(
// Start consuming from EDU api // Start consuming from EDU api
func (s *OutputTypingEventConsumer) Start() error { func (s *OutputTypingEventConsumer) Start() error {
_, err := s.jetstream.Subscribe(s.topic, s.onMessage) _, err := s.jetstream.Subscribe(s.topic, s.onMessage, s.durable)
return err return err
} }

View File

@ -38,6 +38,7 @@ type OutputRoomEventConsumer struct {
cfg *config.SyncAPI cfg *config.SyncAPI
rsAPI api.RoomserverInternalAPI rsAPI api.RoomserverInternalAPI
jetstream nats.JetStreamContext jetstream nats.JetStreamContext
durable nats.SubOpt
topic string topic string
db storage.Database db storage.Database
pduStream types.StreamProvider pduStream types.StreamProvider
@ -61,6 +62,7 @@ func NewOutputRoomEventConsumer(
cfg: cfg, cfg: cfg,
jetstream: js, jetstream: js,
topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputRoomEvent), topic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputRoomEvent),
durable: cfg.Matrix.JetStream.Durable("SyncAPIRoomServerConsumer"),
db: store, db: store,
notifier: notifier, notifier: notifier,
pduStream: pduStream, pduStream: pduStream,
@ -71,7 +73,7 @@ func NewOutputRoomEventConsumer(
// Start consuming from room servers // Start consuming from room servers
func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) Start() error {
_, err := s.jetstream.Subscribe(s.topic, s.onMessage) _, err := s.jetstream.Subscribe(s.topic, s.onMessage, s.durable)
return err return err
} }