From 805a74892efcd18fdcaa14a68a1ba7dfa9486172 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 Jan 2021 14:16:59 +0000 Subject: [PATCH] DNS caching (#1728) * Allow configuring DNS cache * Update sample configs * Fix build errors * Fix time resolution * Default 5m * In seconds * Use WithDNScache * Correct field name * Update go.mod/go.sum to matrix-org/gomatrixserverlib#251 --- build/docker/config/dendrite-config.yaml | 11 ++++++++++ dendrite-config.yaml | 11 ++++++++++ go.mod | 2 +- go.sum | 4 ++-- setup/base.go | 28 ++++++++++++++++++++++-- setup/config/config_global.go | 25 +++++++++++++++++++++ 6 files changed, 76 insertions(+), 5 deletions(-) diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 94dcd992..ca59ae03 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -91,6 +91,17 @@ global: username: metrics password: metrics + # DNS cache options. The DNS cache may reduce the load on DNS servers + # if there is no local caching resolver available for use. + dns_cache: + # Whether or not the DNS cache is enabled. + enabled: false + + # Maximum number of entries to hold in the DNS cache, and + # for how long those items should be considered valid in seconds. + cache_size: 256 + cache_lifetime: 300 + # Configuration for the Appservice API. app_service_api: internal_api: diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 978b1800..a3d1065d 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -103,6 +103,17 @@ global: username: metrics password: metrics + # DNS cache options. The DNS cache may reduce the load on DNS servers + # if there is no local caching resolver available for use. + dns_cache: + # Whether or not the DNS cache is enabled. + enabled: false + + # Maximum number of entries to hold in the DNS cache, and + # for how long those items should be considered valid in seconds. + cache_size: 256 + cache_lifetime: 300 + # Configuration for the Appservice API. app_service_api: internal_api: diff --git a/go.mod b/go.mod index bf12c5ff..a0fee103 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20210121135706-63c66d16c96a + github.com/matrix-org/gomatrixserverlib v0.0.0-20210122141547-5c2c047bfdcd github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index ed24a077..7accf175 100644 --- a/go.sum +++ b/go.sum @@ -567,8 +567,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210121135706-63c66d16c96a h1:cEuy0TSs4F6tYzgsi77cjx3FMBADsU7WWl+USUztD0k= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210121135706-63c66d16c96a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210122141547-5c2c047bfdcd h1:SbWP7aHgACfAQpI2te6LzxK3l0+UqPOb8NOCOxzwV1c= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210122141547-5c2c047bfdcd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/setup/base.go b/setup/base.go index acbf2d35..021dd2b3 100644 --- a/setup/base.go +++ b/setup/base.go @@ -73,6 +73,7 @@ type BaseDendrite struct { httpClient *http.Client Cfg *config.Dendrite Caches *caching.Caches + DNSCache *gomatrixserverlib.DNSCache // KafkaConsumer sarama.Consumer // KafkaProducer sarama.SyncProducer } @@ -111,6 +112,20 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } + var dnsCache *gomatrixserverlib.DNSCache + if cfg.Global.DNSCache.Enabled { + lifetime := time.Second * cfg.Global.DNSCache.CacheLifetime + dnsCache = gomatrixserverlib.NewDNSCache( + cfg.Global.DNSCache.CacheSize, + lifetime, + ) + logrus.Infof( + "DNS cache enabled (size %d, lifetime %s)", + cfg.Global.DNSCache.CacheSize, + lifetime, + ) + } + apiClient := http.Client{ Timeout: time.Minute * 10, Transport: &http2.Transport{ @@ -152,6 +167,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo tracerCloser: closer, Cfg: cfg, Caches: cache, + DNSCache: dnsCache, PublicClientAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicClientPathPrefix).Subrouter().UseEncodedPath(), PublicFederationAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath(), PublicKeyAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(), @@ -252,8 +268,12 @@ func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { if b.Cfg.Global.DisableFederation { return gomatrixserverlib.NewClientWithTransport(noOpHTTPTransport) } + opts := []interface{}{} + if b.Cfg.Global.DNSCache.Enabled { + opts = append(opts, gomatrixserverlib.WithDNSCache{DNSCache: b.DNSCache}) + } client := gomatrixserverlib.NewClient( - b.Cfg.FederationSender.DisableTLSValidation, + b.Cfg.FederationSender.DisableTLSValidation, opts..., ) client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) return client @@ -268,9 +288,13 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli b.Cfg.FederationSender.DisableTLSValidation, noOpHTTPTransport, ) } + opts := []interface{}{} + if b.Cfg.Global.DNSCache.Enabled { + opts = append(opts, gomatrixserverlib.WithDNSCache{DNSCache: b.DNSCache}) + } client := gomatrixserverlib.NewFederationClientWithTimeout( b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey, - b.Cfg.FederationSender.DisableTLSValidation, time.Minute*5, + b.Cfg.FederationSender.DisableTLSValidation, time.Minute*5, opts..., ) client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) return client diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 95652217..d4b068db 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -48,6 +48,9 @@ type Global struct { // Metrics configuration Metrics Metrics `yaml:"metrics"` + + // DNS caching options for all outbound HTTP requests + DNSCache DNSCacheOptions `yaml:"dns_cache"` } func (c *Global) Defaults() { @@ -59,6 +62,7 @@ func (c *Global) Defaults() { c.Kafka.Defaults() c.Metrics.Defaults() + c.DNSCache.Defaults() } func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { @@ -67,6 +71,7 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { c.Kafka.Verify(configErrs, isMonolith) c.Metrics.Verify(configErrs, isMonolith) + c.DNSCache.Verify(configErrs, isMonolith) } type OldVerifyKeys struct { @@ -140,3 +145,23 @@ func (c DatabaseOptions) MaxOpenConns() int { func (c DatabaseOptions) ConnMaxLifetime() time.Duration { return time.Duration(c.ConnMaxLifetimeSeconds) * time.Second } + +type DNSCacheOptions struct { + // Whether the DNS cache is enabled or not + Enabled bool `yaml:"enabled"` + // How many entries to store in the DNS cache at a given time + CacheSize int `yaml:"cache_size"` + // How long a cache entry should be considered valid for + CacheLifetime time.Duration `yaml:"cache_lifetime"` +} + +func (c *DNSCacheOptions) Defaults() { + c.Enabled = false + c.CacheSize = 256 + c.CacheLifetime = time.Minute * 5 +} + +func (c *DNSCacheOptions) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkPositive(configErrs, "cache_size", int64(c.CacheSize)) + checkPositive(configErrs, "cache_lifetime", int64(c.CacheLifetime)) +}