HTTP connection keepalives (#2730)

Beforehand we disabled HTTP keepalives to prevent ambient system
resources from being used by excess idle connections. Now that we've
fixed some bugs in the federation API and device list updater, this
situation is now much better and we don't open so many remote
connections anyway.

Keepalives allow us to not have to handshake TLS so often (which is
quite expensive) and reusing an idle connection is much faster than
having to open a new one. This can help with response times when talking
to remote federated servers.

This PR also adds a new option to disable keepalives if needed:

```
  # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
  # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
  # more quickly than opening new connections each time. Disabling keepalives will close
  # HTTP connections immediately after a successful request but may result in more CPU and
  # memory being used on TLS handshakes for each new connection instead.
  disable_http_keepalives: false
```

dendrite-sample.monolith.yaml

@@ -212,6 +212,13 @@ federation_api:
   # enable this option in production as it presents a security risk!
   disable_tls_validation: false
 
+  # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
+  # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
+  # more quickly than opening new connections each time. Disabling keepalives will close
+  # HTTP connections immediately after a successful request but may result in more CPU and
+  # memory being used on TLS handshakes for each new connection instead.
+  disable_http_keepalives: false
+
   # Perspective keyservers to use as a backup when direct key fetches fail. This may
   # be required to satisfy key requests for servers that are no longer online when
   # joining some rooms.

dendrite-sample.polylith.yaml

@@ -219,6 +219,13 @@ federation_api:
   # enable this option in production as it presents a security risk!
   disable_tls_validation: false
 
+  # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
+  # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
+  # more quickly than opening new connections each time. Disabling keepalives will close
+  # HTTP connections immediately after a successful request but may result in more CPU and
+  # memory being used on TLS handshakes for each new connection instead.
+  disable_http_keepalives: false
+
   # Perspective keyservers to use as a backup when direct key fetches fail. This may
   # be required to satisfy key requests for servers that are no longer online when
   # joining some rooms.

setup/base/base.go

@@ -373,6 +373,7 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli
 	opts := []gomatrixserverlib.ClientOption{
 		gomatrixserverlib.WithTimeout(time.Minute * 5),
 		gomatrixserverlib.WithSkipVerify(b.Cfg.FederationAPI.DisableTLSValidation),
+		gomatrixserverlib.WithKeepAlives(!b.Cfg.FederationAPI.DisableHTTPKeepalives),
 	}
 	if b.Cfg.Global.DNSCache.Enabled {
 		opts = append(opts, gomatrixserverlib.WithDNSCache(b.DNSCache))

setup/config/config_federationapi.go

@@ -22,6 +22,11 @@ type FederationAPI struct {
 	// on remote federation endpoints. This is not recommended in production!
 	DisableTLSValidation bool `yaml:"disable_tls_validation"`
 
+	// DisableHTTPKeepalives prevents Dendrite from keeping HTTP connections
+	// open for reuse for future requests. Connections will be closed quicker
+	// but we may spend more time on TLS handshakes instead.
+	DisableHTTPKeepalives bool `yaml:"disable_http_keepalives"`
+
 	// Perspective keyservers, to use as a backup when direct key fetch
 	// requests don't succeed
 	KeyPerspectives KeyPerspectives `yaml:"key_perspectives"`
@@ -39,6 +44,7 @@ func (c *FederationAPI) Defaults(opts DefaultOpts) {
 	}
 	c.FederationMaxRetries = 16
 	c.DisableTLSValidation = false
+	c.DisableHTTPKeepalives = false
 	if opts.Generate {
 		c.KeyPerspectives = KeyPerspectives{
 			{