From f7cfa758864cc9849ce8b0539895283417b75eb0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 1 May 2020 13:34:53 +0100 Subject: [PATCH] Limit database connections (#980, #564) (#998) * Limit database connections (#564) - Add new options to the config file database: max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 - Implement connection parameter setup on the *DB (database/sql) in internal/sqlutil/trace.go:Open() - Propagate the values in the form of DbProperties interface via all the Open() and NewDatabase() functions Signed-off-by: Tomas Jirka * Fix wasm builds * Remove file accidentally added from working tree Co-authored-by: Tomas Jirka --- appservice/appservice.go | 2 +- appservice/storage/postgres/storage.go | 5 ++- appservice/storage/sqlite3/storage.go | 2 +- appservice/storage/storage.go | 11 +++-- appservice/storage/storage_wasm.go | 6 ++- .../auth/storage/accounts/postgres/storage.go | 4 +- .../auth/storage/accounts/sqlite3/storage.go | 2 +- clientapi/auth/storage/accounts/storage.go | 11 +++-- .../auth/storage/accounts/storage_wasm.go | 7 +++- .../auth/storage/devices/postgres/storage.go | 4 +- .../auth/storage/devices/sqlite3/storage.go | 2 +- clientapi/auth/storage/devices/storage.go | 11 +++-- .../auth/storage/devices/storage_wasm.go | 7 +++- cmd/create-account/main.go | 4 +- cmd/dendrite-demo-libp2p/main.go | 1 + .../storage/postgreswithdht/storage.go | 2 +- .../storage/postgreswithpubsub/storage.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-public-rooms-api-server/main.go | 2 +- common/basecomponent/base.go | 9 ++-- common/config/config.go | 42 +++++++++++++++++++ common/keydb/keydb.go | 8 ++-- common/keydb/keydb_wasm.go | 2 + common/keydb/postgres/keydb.go | 4 +- common/keydb/sqlite3/keydb.go | 2 +- common/sql.go | 8 ++++ dendrite-config.yaml | 3 ++ federationsender/federationsender.go | 2 +- federationsender/storage/postgres/storage.go | 4 +- federationsender/storage/sqlite3/storage.go | 2 +- federationsender/storage/storage.go | 9 ++-- federationsender/storage/storage_wasm.go | 6 ++- internal/sqlutil/trace.go | 21 +++++++++- mediaapi/mediaapi.go | 2 +- mediaapi/storage/postgres/storage.go | 5 ++- mediaapi/storage/sqlite3/storage.go | 2 +- mediaapi/storage/storage.go | 9 ++-- mediaapi/storage/storage_wasm.go | 6 ++- publicroomsapi/storage/postgres/storage.go | 4 +- publicroomsapi/storage/sqlite3/storage.go | 2 +- publicroomsapi/storage/storage.go | 9 ++-- roomserver/roomserver.go | 2 +- roomserver/storage/postgres/storage.go | 5 ++- roomserver/storage/sqlite3/storage.go | 4 +- roomserver/storage/storage.go | 11 ++--- roomserver/storage/storage_wasm.go | 6 ++- syncapi/storage/postgres/syncserver.go | 4 +- syncapi/storage/sqlite3/syncserver.go | 2 +- syncapi/storage/storage.go | 11 ++--- syncapi/storage/storage_wasm.go | 6 ++- syncapi/syncapi.go | 2 +- 51 files changed, 213 insertions(+), 88 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 71d13199..e52db2c2 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -48,7 +48,7 @@ func SetupAppServiceAPIComponent( transactionsCache *transactions.Cache, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB - appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService)) + appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to appservice db") } diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index e145eeee..475db6fc 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import postgres database driver _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -33,10 +34,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 0cd1e4ab..0b0590f6 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index 9fbd2a1f..bf0a9b0c 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -21,19 +21,22 @@ import ( "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" + "github.com/matrix-org/dendrite/common" ) -func NewDatabase(dataSourceName string) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets DB connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) case "file": return sqlite3.NewDatabase(dataSourceName) default: - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } } diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index 2bd1433f..ff0330ae 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -19,9 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" + "github.com/matrix-org/dendrite/common" ) -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 8ce367a3..7e2073ec 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -44,10 +44,10 @@ type Database struct { } // NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index e190ba6c..30a93e7e 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -50,7 +50,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index c643a4d0..394cc5e1 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -21,20 +21,23 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets postgres connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) case "file": return sqlite3.NewDatabase(dataSourceName, serverName) default: - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } } diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 828afc6b..61f9d699 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -19,10 +19,15 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam + serverName gomatrixserverlib.ServerName, +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index 3f613cf3..57c268df 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -36,10 +36,10 @@ type Database struct { } // NewDatabase creates a new device database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index 85a8def2..b69d6278 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -41,7 +41,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 99211db2..ec47a327 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -21,20 +21,23 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets postgres connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) case "file": return sqlite3.NewDatabase(dataSourceName, serverName) default: - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } } diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index 32285288..e25b7c64 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -19,10 +19,15 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam + serverName gomatrixserverlib.ServerName, +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index fc51a5bb..eca9b2fe 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -63,7 +63,7 @@ func main() { serverName := gomatrixserverlib.ServerName(*serverNameStr) - accountDB, err := accounts.NewDatabase(*database, serverName) + accountDB, err := accounts.NewDatabase(*database, nil, serverName) if err != nil { fmt.Println(err.Error()) os.Exit(1) @@ -78,7 +78,7 @@ func main() { os.Exit(1) } - deviceDB, err := devices.NewDatabase(*database, serverName) + deviceDB, err := devices.NewDatabase(*database, nil, serverName) if err != nil { fmt.Println(err.Error()) os.Exit(1) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 28c7153f..0ff610a5 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -56,6 +56,7 @@ func createKeyDB( ) keydb.Database { db, err := keydb.NewDatabase( string(base.Base.Cfg.Database.ServerKey), + base.Base.Cfg.DbProperties(), base.Base.Cfg.Matrix.ServerName, base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), base.Base.Cfg.Matrix.KeyID, diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go index 819469ee..cd2804c9 100644 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go @@ -45,7 +45,7 @@ type PublicRoomsServerDatabase struct { // NewPublicRoomsServerDatabase creates a new public rooms server database. func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName) + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil) if err != nil { return nil, err } diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go index 66119224..e4372c64 100644 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go @@ -48,7 +48,7 @@ type PublicRoomsServerDatabase struct { // NewPublicRoomsServerDatabase creates a new public rooms server database. func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName) + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil) if err != nil { return nil, err } diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 06001971..e004bc12 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -79,7 +79,7 @@ func main() { eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index fca39a2f..c3b49f4f 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -32,7 +32,7 @@ func main() { rsAPI := base.CreateHTTPRoomserverAPIs() rsAPI.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 154acd80..a7e6736a 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -150,7 +150,7 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() devices.Database { - db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.Matrix.ServerName) + db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to devices db") } @@ -161,7 +161,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database { // CreateAccountsDB creates a new instance of the accounts database. Should only // be called once per component. func (b *BaseDendrite) CreateAccountsDB() accounts.Database { - db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.Matrix.ServerName) + db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") } @@ -174,6 +174,7 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { func (b *BaseDendrite) CreateKeyDB() keydb.Database { db, err := keydb.NewDatabase( string(b.Cfg.Database.ServerKey), + b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName, b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), b.Cfg.Matrix.KeyID, @@ -244,7 +245,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) + db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } @@ -254,7 +255,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { logrus.WithError(err).Panic("Failed to setup naffka database") } } else { - db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka)) + db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/config/config.go b/common/config/config.go index 6b61fda7..9a29186a 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -188,6 +188,12 @@ type Dendrite struct { PublicRoomsAPI DataSource `yaml:"public_rooms_api"` // The Naffka database is used internally by the naffka library, if used. Naffka DataSource `yaml:"naffka,omitempty"` + // Maximum open connections to the DB (0 = use default, negative means unlimited) + MaxOpenConns int `yaml:"max_open_conns"` + // Maximum idle connections to the DB (0 = use default, negative means unlimited) + MaxIdleConns int `yaml:"max_idle_conns"` + // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) + ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"` } `yaml:"database"` // TURN Server Config @@ -484,6 +490,15 @@ func (config *Dendrite) SetDefaults() { defaultMaxFileSizeBytes := FileSizeBytes(10485760) config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes } + + if config.Database.MaxIdleConns == 0 { + config.Database.MaxIdleConns = 2 + } + + if config.Database.MaxOpenConns == 0 { + config.Database.MaxOpenConns = 100 + } + } // Error returns a string detailing how many errors were contained within a @@ -746,6 +761,33 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err ) } +// MaxIdleConns returns maximum idle connections to the DB +func (config Dendrite) MaxIdleConns() int { + return config.Database.MaxIdleConns +} + +// MaxOpenConns returns maximum open connections to the DB +func (config Dendrite) MaxOpenConns() int { + return config.Database.MaxOpenConns +} + +// ConnMaxLifetime returns maximum amount of time a connection may be reused +func (config Dendrite) ConnMaxLifetime() time.Duration { + return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second +} + +// DbProperties functions return properties used by database/sql/DB +type DbProperties interface { + MaxIdleConns() int + MaxOpenConns() int + ConnMaxLifetime() time.Duration +} + +// DbProperties returns cfg as a DbProperties interface +func (config Dendrite) DbProperties() DbProperties { + return config +} + // logrusLogger is a small wrapper that implements jaeger.Logger using logrus. type logrusLogger struct { l *logrus.Logger diff --git a/common/keydb/keydb.go b/common/keydb/keydb.go index fe6d87fc..397d7849 100644 --- a/common/keydb/keydb.go +++ b/common/keydb/keydb.go @@ -21,6 +21,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/keydb/postgres" "github.com/matrix-org/dendrite/common/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -29,20 +30,21 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) case "file": return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) default: - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) } } diff --git a/common/keydb/keydb_wasm.go b/common/keydb/keydb_wasm.go index 807ed40b..38e59582 100644 --- a/common/keydb/keydb_wasm.go +++ b/common/keydb/keydb_wasm.go @@ -20,6 +20,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) @@ -27,6 +28,7 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/common/keydb/postgres/keydb.go b/common/keydb/postgres/keydb.go index 2879683e..6149d877 100644 --- a/common/keydb/postgres/keydb.go +++ b/common/keydb/postgres/keydb.go @@ -21,6 +21,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -37,11 +38,12 @@ type Database struct { // Returns an error if there was a problem talking to the database. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open("postgres", dataSourceName) + db, err := sqlutil.Open("postgres", dataSourceName, dbProperties) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 82d2a491..1405836a 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -44,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName) + db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil) if err != nil { return nil, err } diff --git a/common/sql.go b/common/sql.go index f50a5896..e93ff1c2 100644 --- a/common/sql.go +++ b/common/sql.go @@ -18,6 +18,7 @@ import ( "database/sql" "fmt" "runtime" + "time" ) // A Transaction is something that can be committed or rolledback. @@ -99,3 +100,10 @@ func SQLiteDriverName() string { } return "sqlite3" } + +// DbProperties functions return properties used by database/sql/DB +type DbProperties interface { + MaxIdleConns() int + MaxOpenConns() int + ConnMaxLifetime() time.Duration +} diff --git a/dendrite-config.yaml b/dendrite-config.yaml index bed78a5a..8c8fba39 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -116,6 +116,9 @@ database: federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable" public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable" + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 # If using naffka you need to specify a naffka database # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 64de4fd2..cf439552 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -37,7 +37,7 @@ func SetupFederationSenderComponent( rsAPI roomserverAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, ) api.FederationSenderInternalAPI { - federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) + federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index b909a189..c3892ac1 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -33,10 +33,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 458d7d7e..77274447 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -38,7 +38,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index 2f018dff..d481e58a 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case "file": return sqlite3.NewDatabase(dataSourceName) case "postgres": - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) default: - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } } diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index f2c8ae1b..44d4c806 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index 3d5fa7dc..42ac4e58 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -21,9 +21,11 @@ import ( "fmt" "io" "os" + "regexp" "strings" "time" + "github.com/matrix-org/dendrite/common" "github.com/ngrok/sqlmw" "github.com/sirupsen/logrus" ) @@ -76,12 +78,27 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [ // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 -func Open(driverName, dsn string) (*sql.DB, error) { +func Open(driverName, dsn string, dbProperties common.DbProperties) (*sql.DB, error) { if tracingEnabled { // install the wrapped driver driverName += "-trace" } - return sql.Open(driverName, dsn) + db, err := sql.Open(driverName, dsn) + if err != nil { + return nil, err + } + if driverName != common.SQLiteDriverName() && dbProperties != nil { + logrus.WithFields(logrus.Fields{ + "MaxOpenConns": dbProperties.MaxOpenConns(), + "MaxIdleConns": dbProperties.MaxIdleConns(), + "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), + "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), + }).Debug("Setting DB connection limits") + db.SetMaxOpenConns(dbProperties.MaxOpenConns()) + db.SetMaxIdleConns(dbProperties.MaxIdleConns()) + db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime()) + } + return db, nil } func init() { diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index f2e614c1..4a0f5d18 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -29,7 +29,7 @@ func SetupMediaAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, ) { - mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI)) + mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index 18126b15..4ddfc8fd 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -33,10 +34,10 @@ type Database struct { } // Open opens a postgres database. -func Open(dataSourceName string) (*Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index abafecf2..8fa2e537 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index c533477c..6589c830 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string) (Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) case "file": return sqlite3.Open(dataSourceName) default: - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } } diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 92f0ad13..3c39e5d3 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string) (Database, error) { +func Open( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 8c4660cc..6242c9d5 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -36,10 +36,10 @@ type PublicRoomsServerDatabase struct { type attributeValue interface{} // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index 12160162..efe35bdd 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -41,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index e674514a..7dcfe563 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -19,6 +19,7 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" ) @@ -27,17 +28,17 @@ const schemePostgres = "postgres" const schemeFile = "file" // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case schemePostgres: - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) case schemeFile: return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) default: - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) } } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 916e25fb..450da5bb 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -35,7 +35,7 @@ func SetupRoomServerComponent( keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, ) api.RoomserverInternalAPI { - roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) + roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 5b5c61b0..1d825ecc 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" // Import the postgres database driver. @@ -36,10 +37,10 @@ type Database struct { } // Open a postgres database. -func Open(dataSourceName string) (*Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index b6e846df..e77fea9c 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { db *sql.DB } -// Open a postgres database. +// Open a sqlite database. func Open(dataSourceName string) (*Database, error) { var d Database uri, err := url.Parse(dataSourceName) @@ -52,7 +52,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 7b9109aa..99e99a00 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) -// NewPublicRoomsServerDatabase opens a database connection. -func Open(dataSourceName string) (Database, error) { +// Open opens a database connection. +func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) case "file": return sqlite3.Open(dataSourceName) default: - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } } diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go index d7fc352e..5fa48bc9 100644 --- a/roomserver/storage/storage_wasm.go +++ b/roomserver/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func Open(dataSourceName string) (Database, error) { +func Open( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index a6de1517..1845ac38 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -61,10 +61,10 @@ type SyncServerDatasource struct { } // NewSyncServerDatasource creates a new sync server database -func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) { +func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProperties) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error - if d.db, err = sqlutil.Open("postgres", dbDataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 7e8e4ff5..425073a7 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -80,7 +80,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.prepare(); err != nil { diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index c56db063..9574f37b 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) -// NewPublicRoomsServerDatabase opens a database connection. -func NewSyncServerDatasource(dataSourceName string) (Database, error) { +// NewSyncServerDatasource opens a database connection. +func NewSyncServerDatasource(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) case "file": return sqlite3.NewSyncServerDatasource(dataSourceName) default: - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) } } diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index 43806a01..84bd9df9 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func NewSyncServerDatasource(dataSourceName string) (Database, error) { +func NewSyncServerDatasource( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 5ab1ec7c..4219d560 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -42,7 +42,7 @@ func SetupSyncAPIComponent( federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, ) { - syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI)) + syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") }