Remove TLS fingerprints, improve perspective unmarshal handling (#1452)

* Add prefer_direct_fetch option

* Update gomatrixserverlib

* Update gomatrixserverlib

* Update gomatrixserverlib

* Don't deal in TLS fingerprints anymore

dendrite-config.yaml

@@ -274,6 +274,11 @@ server_key_api:
     - key_id: ed25519:a_RXGa
       public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
 
+  # This option will control whether Dendrite will prefer to look up keys directly
+  # or whether it should try perspective servers first, using direct fetches as a
+  # last resort.
+  prefer_direct_fetch: false
+
 # Configuration for the Sync API.
 sync_api:
   internal_api:

federationapi/routing/keys.go

@@ -136,7 +136,6 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver
 	var keys gomatrixserverlib.ServerKeys
 
 	keys.ServerName = cfg.Matrix.ServerName
-	keys.TLSFingerprints = cfg.TLSFingerPrints
 	keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(validUntil)
 
 	publicKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200925165243-b9780a852681
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a
 	github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.2

go.sum

@@ -569,8 +569,14 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200925165243-b9780a852681 h1:75fM7vPHiFGt+XxktT17LJD972XMtJ1n7FU1MpC08Zc=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad h1:n0P/Oy8ZqqTPzum6FEayAjamsmvJTuIcA10WQ8GcK70=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200925165243-b9780a852681/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553 h1:tiel2c3I9xr0SRS05g3UvOjj6Sgg1I3Yn2/oGA1GgLk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2 h1:a07U5eFT521mFiUtA/A8NwiZp5vfRU59/QKs+pa3Fkc=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a h1:kIwbS7eY7P/MX0oN4wRHGkoc4eTTnwOcdCawBZ3SrJI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

internal/config/config.go

@@ -16,7 +16,6 @@ package config
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"encoding/pem"
 	"fmt"
 	"io"
@@ -252,20 +251,6 @@ func loadConfig(
 		c.Global.OldVerifyKeys[i].KeyID, c.Global.OldVerifyKeys[i].PrivateKey = keyID, privateKey
 	}
 
-	for _, certPath := range c.FederationAPI.FederationCertificatePaths {
-		absCertPath := absPath(basePath, certPath)
-		var pemData []byte
-		pemData, err = readFile(absCertPath)
-		if err != nil {
-			return nil, err
-		}
-		fingerprint := fingerprintPEM(pemData)
-		if fingerprint == nil {
-			return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath)
-		}
-		c.FederationAPI.TLSFingerPrints = append(c.FederationAPI.TLSFingerPrints, *fingerprint)
-	}
-
 	c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath))
 
 	// Generate data from config options
@@ -494,20 +479,6 @@ func readKeyPEM(path string, data []byte, enforceKeyIDFormat bool) (gomatrixserv
 	}
 }
 
-func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint {
-	for {
-		var certDERBlock *pem.Block
-		certDERBlock, data = pem.Decode(data)
-		if data == nil {
-			return nil
-		}
-		if certDERBlock.Type == "CERTIFICATE" {
-			digest := sha256.Sum256(certDERBlock.Bytes)
-			return &gomatrixserverlib.TLSFingerprint{SHA256: digest[:]}
-		}
-	}
-}
-
 // AppServiceURL returns a HTTP URL for where the appservice component is listening.
 func (config *Dendrite) AppServiceURL() string {
 	// Hard code the appservice server to talk HTTP for now.

internal/config/config_federationapi.go

@@ -1,7 +1,5 @@
 package config
 
-import "github.com/matrix-org/gomatrixserverlib"
-
 type FederationAPI struct {
 	Matrix *Global `yaml:"-"`
 
@@ -14,10 +12,6 @@ type FederationAPI struct {
 	// to match one of these certificates.
 	// The certificates should be in PEM format.
 	FederationCertificatePaths []Path `yaml:"federation_certificates"`
-
-	// A list of SHA256 TLS fingerprints for the X509 certificates used by the
-	// federation listener for this server.
-	TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"`
 }
 
 func (c *FederationAPI) Defaults() {

internal/config/config_serverkey.go

@@ -14,6 +14,9 @@ type ServerKeyAPI struct {
 	// Perspective keyservers, to use as a backup when direct key fetch
 	// requests don't succeed
 	KeyPerspectives KeyPerspectives `yaml:"key_perspectives"`
+
+	// Should we prefer direct key fetches over perspective ones?
+	PreferDirectFetch bool `yaml:"prefer_direct_fetch"`
 }
 
 func (c *ServerKeyAPI) Defaults() {

internal/config/config_test.go

@@ -253,19 +253,6 @@ Key-ID: ` + testKeyID + `
 -----END MATRIX PRIVATE KEY-----
 `
 
-func TestFingerprintPEM(t *testing.T) {
-	got := fingerprintPEM([]byte(testCert))
-	if got == nil {
-		t.Error("failed to calculate fingerprint")
-	}
-	if string(got.SHA256) != testCertFingerprint {
-		t.Errorf("bad fingerprint: wanted %q got %q", got, testCertFingerprint)
-	}
-
-}
-
-const testCertFingerprint = "56.\\SPQxE\xd4\x95\xfb\xf6\xd5\x04\x91\xcb/\x07\xb1^\x88\x08\xe3\xc1p\xdfY\x04\x19w\xcb"
-
 const testCert = `
 -----BEGIN CERTIFICATE-----
 MIIE0zCCArugAwIBAgIJAPype3u24LJeMA0GCSqGSIb3DQEBCwUAMAAwHhcNMTcw

serverkeyapi/internal/api.go

@@ -191,7 +191,7 @@ func (s *ServerKeyAPI) handleFetcherKeys(
 	// Try to fetch the keys.
 	fetcherResults, err := fetcher.FetchKeys(fetcherCtx, requests)
 	if err != nil {
-		return err
+		return fmt.Errorf("fetcher.FetchKeys: %w", err)
 	}
 
 	// Build a map of the results that we want to commit to the

serverkeyapi/serverkeyapi.go

@@ -51,13 +51,24 @@ func NewInternalAPI(
 		ServerKeyValidity: cfg.Matrix.KeyValidityPeriod,
 		FedClient:         fedClient,
 		OurKeyRing: gomatrixserverlib.KeyRing{
-			KeyFetchers: []gomatrixserverlib.KeyFetcher{
+			KeyFetchers: []gomatrixserverlib.KeyFetcher{},
+			KeyDatabase: serverKeyDB,
+		},
+	}
+
+	addDirectFetcher := func() {
+		internalAPI.OurKeyRing.KeyFetchers = append(
+			internalAPI.OurKeyRing.KeyFetchers,
 			&gomatrixserverlib.DirectKeyFetcher{
 				Client: fedClient,
 			},
-			},
+		)
-			KeyDatabase: serverKeyDB,
+	}
-		},
+
+	if cfg.PreferDirectFetch {
+		addDirectFetcher()
+	} else {
+		defer addDirectFetcher()
 	}
 
 	var b64e = base64.StdEncoding.WithPadding(base64.NoPadding)