From 4a7fb9c045211c54c13610119a0f5ed0df355a0f Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Oct 2020 18:09:41 +0100 Subject: [PATCH] Automatically upgrade databases on startup (#1529) * Support auto-upgrading accounts DB * Auto-upgrade device DB deltas * Support up/downgrading from cmd/goose * Linting * Create tables then do migrations then prepare statements To avoid failing due to some things not existing * Linting --- cmd/goose/main.go | 86 ++++++++++-- internal/sqlutil/migrate.go | 130 ++++++++++++++++++ .../accounts/postgres/accounts_table.go | 9 +- .../deltas/20200929203058_is_active.go | 33 +++++ .../deltas/20200929203058_is_active.sql | 9 -- userapi/storage/accounts/postgres/storage.go | 15 ++ .../accounts/sqlite3/accounts_table.go | 10 +- .../deltas/20200929203058_is_active.go | 64 +++++++++ .../deltas/20200929203058_is_active.sql | 38 ----- userapi/storage/accounts/sqlite3/storage.go | 14 ++ .../deltas/20201001204705_last_seen_ts_ip.go | 39 ++++++ .../deltas/20201001204705_last_seen_ts_ip.sql | 13 -- .../storage/devices/postgres/devices_table.go | 9 +- userapi/storage/devices/postgres/storage.go | 14 ++ .../deltas/20201001204705_last_seen_ts_ip.go | 70 ++++++++++ .../deltas/20201001204705_last_seen_ts_ip.sql | 44 ------ .../storage/devices/sqlite3/devices_table.go | 9 +- userapi/storage/devices/sqlite3/storage.go | 12 ++ 18 files changed, 485 insertions(+), 133 deletions(-) create mode 100644 internal/sqlutil/migrate.go create mode 100644 userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go delete mode 100644 userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql create mode 100644 userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go delete mode 100644 userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql create mode 100644 userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go delete mode 100644 userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql create mode 100644 userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.go delete mode 100644 userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql diff --git a/cmd/goose/main.go b/cmd/goose/main.go index ef3942d9..83c97a72 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -8,19 +8,38 @@ import ( "log" "os" - // Example complex Go migration import: - // _ "github.com/matrix-org/dendrite/serverkeyapi/storage/postgres/deltas" + pgaccounts "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" + slaccounts "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas" + pgdevices "github.com/matrix-org/dendrite/userapi/storage/devices/postgres/deltas" + sldevices "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3/deltas" "github.com/pressly/goose" _ "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" ) -var ( - flags = flag.NewFlagSet("goose", flag.ExitOnError) - dir = flags.String("dir", ".", "directory with migration files") +const ( + AppService = "appservice" + FederationSender = "federationsender" + KeyServer = "keyserver" + MediaAPI = "mediaapi" + RoomServer = "roomserver" + SigningKeyServer = "signingkeyserver" + SyncAPI = "syncapi" + UserAPIAccounts = "userapi_accounts" + UserAPIDevices = "userapi_devices" ) +var ( + dir = flags.String("dir", "", "directory with migration files") + flags = flag.NewFlagSet("goose", flag.ExitOnError) + component = flags.String("component", "", "dendrite component name") + knownDBs = []string{ + AppService, FederationSender, KeyServer, MediaAPI, RoomServer, SigningKeyServer, SyncAPI, UserAPIAccounts, UserAPIDevices, + } +) + +// nolint: gocyclo func main() { err := flags.Parse(os.Args[1:]) if err != nil { @@ -37,19 +56,20 @@ Drivers: sqlite3 Examples: - goose -d roomserver/storage/sqlite3/deltas sqlite3 ./roomserver.db status - goose -d roomserver/storage/sqlite3/deltas sqlite3 ./roomserver.db up + goose -component roomserver sqlite3 ./roomserver.db status + goose -component roomserver sqlite3 ./roomserver.db up - goose -d roomserver/storage/postgres/deltas postgres "user=dendrite dbname=dendrite sslmode=disable" status + goose -component roomserver postgres "user=dendrite dbname=dendrite sslmode=disable" status Options: - - -dir string - directory with migration files (default ".") + -component string + Dendrite component name e.g roomserver, signingkeyserver, clientapi, syncapi -table string migrations table name (default "goose_db_version") -h print help -v enable verbose mode + -dir string + directory with migration files, only relevant when creating new migrations. -version print version @@ -74,6 +94,25 @@ Commands: fmt.Println("engine must be one of 'sqlite3' or 'postgres'") return } + + knownComponent := false + for _, c := range knownDBs { + if c == *component { + knownComponent = true + break + } + } + if !knownComponent { + fmt.Printf("component must be one of %v\n", knownDBs) + return + } + + if engine == "sqlite3" { + loadSQLiteDeltas(*component) + } else { + loadPostgresDeltas(*component) + } + dbstring, command := args[1], args[2] db, err := goose.OpenDBWithDriver(engine, dbstring) @@ -92,7 +131,30 @@ Commands: arguments = append(arguments, args[3:]...) } - if err := goose.Run(command, db, *dir, arguments...); err != nil { + // goose demands a directory even though we don't use it for upgrades + d := *dir + if d == "" { + d = os.TempDir() + } + if err := goose.Run(command, db, d, arguments...); err != nil { log.Fatalf("goose %v: %v", command, err) } } + +func loadSQLiteDeltas(component string) { + switch component { + case UserAPIAccounts: + slaccounts.LoadFromGoose() + case UserAPIDevices: + sldevices.LoadFromGoose() + } +} + +func loadPostgresDeltas(component string) { + switch component { + case UserAPIAccounts: + pgaccounts.LoadFromGoose() + case UserAPIDevices: + pgdevices.LoadFromGoose() + } +} diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go new file mode 100644 index 00000000..833977ba --- /dev/null +++ b/internal/sqlutil/migrate.go @@ -0,0 +1,130 @@ +package sqlutil + +import ( + "database/sql" + "fmt" + "runtime" + "sort" + + "github.com/matrix-org/dendrite/internal/config" + "github.com/pressly/goose" +) + +type Migrations struct { + registeredGoMigrations map[int64]*goose.Migration +} + +func NewMigrations() *Migrations { + return &Migrations{ + registeredGoMigrations: make(map[int64]*goose.Migration), + } +} + +// Copy-pasted from goose directly to store migrations into a map we control + +// AddMigration adds a migration. +func (m *Migrations) AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) { + _, filename, _, _ := runtime.Caller(1) + m.AddNamedMigration(filename, up, down) +} + +// AddNamedMigration : Add a named migration. +func (m *Migrations) AddNamedMigration(filename string, up func(*sql.Tx) error, down func(*sql.Tx) error) { + v, _ := goose.NumericComponent(filename) + migration := &goose.Migration{Version: v, Next: -1, Previous: -1, Registered: true, UpFn: up, DownFn: down, Source: filename} + + if existing, ok := m.registeredGoMigrations[v]; ok { + panic(fmt.Sprintf("failed to add migration %q: version conflicts with %q", filename, existing.Source)) + } + + m.registeredGoMigrations[v] = migration +} + +// RunDeltas up to the latest version. +func (m *Migrations) RunDeltas(db *sql.DB, props *config.DatabaseOptions) error { + maxVer := goose.MaxVersion + minVer := int64(0) + migrations, err := m.collect(minVer, maxVer) + if err != nil { + return fmt.Errorf("RunDeltas: Failed to collect migrations: %w", err) + } + if props.ConnectionString.IsPostgres() { + if err = goose.SetDialect("postgres"); err != nil { + return err + } + } else if props.ConnectionString.IsSQLite() { + if err = goose.SetDialect("sqlite3"); err != nil { + return err + } + } else { + return fmt.Errorf("Unknown connection string: %s", props.ConnectionString) + } + for { + current, err := goose.EnsureDBVersion(db) + if err != nil { + return fmt.Errorf("RunDeltas: Failed to EnsureDBVersion: %w", err) + } + + next, err := migrations.Next(current) + if err != nil { + if err == goose.ErrNoNextVersion { + return nil + } + + return fmt.Errorf("RunDeltas: Failed to load next migration to %+v : %w", next, err) + } + + if err = next.Up(db); err != nil { + return fmt.Errorf("RunDeltas: Failed run migration: %w", err) + } + } +} + +func (m *Migrations) collect(current, target int64) (goose.Migrations, error) { + var migrations goose.Migrations + + // Go migrations registered via goose.AddMigration(). + for _, migration := range m.registeredGoMigrations { + v, err := goose.NumericComponent(migration.Source) + if err != nil { + return nil, err + } + if versionFilter(v, current, target) { + migrations = append(migrations, migration) + } + } + + migrations = sortAndConnectMigrations(migrations) + + return migrations, nil +} + +func sortAndConnectMigrations(migrations goose.Migrations) goose.Migrations { + sort.Sort(migrations) + + // now that we're sorted in the appropriate direction, + // populate next and previous for each migration + for i, m := range migrations { + prev := int64(-1) + if i > 0 { + prev = migrations[i-1].Version + migrations[i-1].Next = m.Version + } + migrations[i].Previous = prev + } + + return migrations +} + +func versionFilter(v, current, target int64) bool { + + if target > current { + return v > current && v <= target + } + + if target < current { + return v <= current && v > target + } + + return false +} diff --git a/userapi/storage/accounts/postgres/accounts_table.go b/userapi/storage/accounts/postgres/accounts_table.go index 254da84c..4eaa5b58 100644 --- a/userapi/storage/accounts/postgres/accounts_table.go +++ b/userapi/storage/accounts/postgres/accounts_table.go @@ -75,11 +75,12 @@ type accountsStatements struct { serverName gomatrixserverlib.ServerName } +func (s *accountsStatements) execSchema(db *sql.DB) error { + _, err := db.Exec(accountsSchema) + return err +} + func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { - _, err = db.Exec(accountsSchema) - if err != nil { - return - } if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil { return } diff --git a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go new file mode 100644 index 00000000..9e14286e --- /dev/null +++ b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go @@ -0,0 +1,33 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/pressly/goose" +) + +func LoadFromGoose() { + goose.AddMigration(UpIsActive, DownIsActive) +} + +func LoadIsActive(m *sqlutil.Migrations) { + m.AddMigration(UpIsActive, DownIsActive) +} + +func UpIsActive(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;") + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownIsActive(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE account_accounts DROP COLUMN is_deactivated;") + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql deleted file mode 100644 index 32e6e166..00000000 --- a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql +++ /dev/null @@ -1,9 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE account_accounts DROP COLUMN is_deactivated; --- +goose StatementEnd diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 2230f7e7..40c4b8ff 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -25,6 +25,8 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" + _ "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -55,6 +57,18 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver db: db, writer: sqlutil.NewDummyWriter(), } + + // Create tables before executing migrations so we don't fail if the table is missing, + // and THEN prepare statements so we don't fail due to referencing new columns + if err = d.accounts.execSchema(db); err != nil { + return nil, err + } + m := sqlutil.NewMigrations() + deltas.LoadIsActive(m) + if err = m.RunDeltas(db, dbProperties); err != nil { + return nil, err + } + if err = d.PartitionOffsetStatements.Prepare(db, d.writer, "account"); err != nil { return nil, err } @@ -70,6 +84,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.threepids.prepare(db); err != nil { return nil, err } + return d, nil } diff --git a/userapi/storage/accounts/sqlite3/accounts_table.go b/userapi/storage/accounts/sqlite3/accounts_table.go index d0ea8a8b..50f07237 100644 --- a/userapi/storage/accounts/sqlite3/accounts_table.go +++ b/userapi/storage/accounts/sqlite3/accounts_table.go @@ -74,13 +74,13 @@ type accountsStatements struct { serverName gomatrixserverlib.ServerName } +func (s *accountsStatements) execSchema(db *sql.DB) error { + _, err := db.Exec(accountsSchema) + return err +} + func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { s.db = db - - _, err = db.Exec(accountsSchema) - if err != nil { - return - } if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil { return } diff --git a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go new file mode 100644 index 00000000..9fddb05a --- /dev/null +++ b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go @@ -0,0 +1,64 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/pressly/goose" +) + +func LoadFromGoose() { + goose.AddMigration(UpIsActive, DownIsActive) +} + +func LoadIsActive(m *sqlutil.Migrations) { + m.AddMigration(UpIsActive, DownIsActive) +} + +func UpIsActive(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE account_accounts RENAME TO account_accounts_tmp; +CREATE TABLE account_accounts ( + localpart TEXT NOT NULL PRIMARY KEY, + created_ts BIGINT NOT NULL, + password_hash TEXT, + appservice_id TEXT, + is_deactivated BOOLEAN DEFAULT 0 +); +INSERT + INTO account_accounts ( + localpart, created_ts, password_hash, appservice_id + ) SELECT + localpart, created_ts, password_hash, appservice_id + FROM account_accounts_tmp +; +DROP TABLE account_accounts_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownIsActive(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE account_accounts RENAME TO account_accounts_tmp; +CREATE TABLE account_accounts ( + localpart TEXT NOT NULL PRIMARY KEY, + created_ts BIGINT NOT NULL, + password_hash TEXT, + appservice_id TEXT +); +INSERT + INTO account_accounts ( + localpart, created_ts, password_hash, appservice_id + ) SELECT + localpart, created_ts, password_hash, appservice_id + FROM account_accounts_tmp +; +DROP TABLE account_accounts_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql deleted file mode 100644 index 51e9bae3..00000000 --- a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql +++ /dev/null @@ -1,38 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( - localpart TEXT NOT NULL PRIMARY KEY, - created_ts BIGINT NOT NULL, - password_hash TEXT, - appservice_id TEXT, - is_deactivated BOOLEAN DEFAULT 0 -); -INSERT - INTO account_accounts ( - localpart, created_ts, password_hash, appservice_id - ) SELECT - localpart, created_ts, password_hash, appservice_id - FROM account_accounts_tmp -; -DROP TABLE account_accounts_tmp; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( - localpart TEXT NOT NULL PRIMARY KEY, - created_ts BIGINT NOT NULL, - password_hash TEXT, - appservice_id TEXT -); -INSERT - INTO account_accounts ( - localpart, created_ts, password_hash, appservice_id - ) SELECT - localpart, created_ts, password_hash, appservice_id - FROM account_accounts_tmp -; -DROP TABLE account_accounts_tmp; --- +goose StatementEnd diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 7a2830a9..0be7bcbe 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" // Import the sqlite3 database driver. @@ -60,6 +61,18 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver db: db, writer: sqlutil.NewExclusiveWriter(), } + + // Create tables before executing migrations so we don't fail if the table is missing, + // and THEN prepare statements so we don't fail due to referencing new columns + if err = d.accounts.execSchema(db); err != nil { + return nil, err + } + m := sqlutil.NewMigrations() + deltas.LoadIsActive(m) + if err = m.RunDeltas(db, dbProperties); err != nil { + return nil, err + } + partitions := sqlutil.PartitionOffsetStatements{} if err = partitions.Prepare(db, d.writer, "account"); err != nil { return nil, err @@ -76,6 +89,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.threepids.prepare(db); err != nil { return nil, err } + return d, nil } diff --git a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go new file mode 100644 index 00000000..290f854c --- /dev/null +++ b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go @@ -0,0 +1,39 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/pressly/goose" +) + +func LoadFromGoose() { + goose.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP) +} + +func LoadLastSeenTSIP(m *sqlutil.Migrations) { + m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP) +} + +func UpLastSeenTSIP(tx *sql.Tx) error { + _, err := tx.Exec(` +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownLastSeenTSIP(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE device_devices DROP COLUMN last_seen_ts; + ALTER TABLE device_devices DROP COLUMN ip; + ALTER TABLE device_devices DROP COLUMN user_agent;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql deleted file mode 100644 index e7900b0b..00000000 --- a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql +++ /dev/null @@ -1,13 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000; -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT; -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE device_devices DROP COLUMN last_seen_ts; -ALTER TABLE device_devices DROP COLUMN ip; -ALTER TABLE device_devices DROP COLUMN user_agent; --- +goose StatementEnd diff --git a/userapi/storage/devices/postgres/devices_table.go b/userapi/storage/devices/postgres/devices_table.go index 2a4d337c..379fed79 100644 --- a/userapi/storage/devices/postgres/devices_table.go +++ b/userapi/storage/devices/postgres/devices_table.go @@ -111,11 +111,12 @@ type devicesStatements struct { serverName gomatrixserverlib.ServerName } +func (s *devicesStatements) execSchema(db *sql.DB) error { + _, err := db.Exec(devicesSchema) + return err +} + func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { - _, err = db.Exec(devicesSchema) - if err != nil { - return - } if s.insertDeviceStmt, err = db.Prepare(insertDeviceSQL); err != nil { return } diff --git a/userapi/storage/devices/postgres/storage.go b/userapi/storage/devices/postgres/storage.go index faa5796b..e318b260 100644 --- a/userapi/storage/devices/postgres/storage.go +++ b/userapi/storage/devices/postgres/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/devices/postgres/deltas" "github.com/matrix-org/gomatrixserverlib" ) @@ -42,9 +43,22 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver return nil, err } d := devicesStatements{} + + // Create tables before executing migrations so we don't fail if the table is missing, + // and THEN prepare statements so we don't fail due to referencing new columns + if err = d.execSchema(db); err != nil { + return nil, err + } + m := sqlutil.NewMigrations() + deltas.LoadLastSeenTSIP(m) + if err = m.RunDeltas(db, dbProperties); err != nil { + return nil, err + } + if err = d.prepare(db, serverName); err != nil { return nil, err } + return &Database{db, d}, nil } diff --git a/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.go new file mode 100644 index 00000000..26209826 --- /dev/null +++ b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.go @@ -0,0 +1,70 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/pressly/goose" +) + +func LoadFromGoose() { + goose.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP) +} + +func LoadLastSeenTSIP(m *sqlutil.Migrations) { + m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP) +} + +func UpLastSeenTSIP(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE device_devices RENAME TO device_devices_tmp; + CREATE TABLE device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + last_seen_ts BIGINT, + ip TEXT, + user_agent TEXT, + UNIQUE (localpart, device_id) + ); + INSERT + INTO device_devices ( + access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent + ) SELECT + access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', '' + FROM device_devices_tmp; + DROP TABLE device_devices_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownLastSeenTSIP(tx *sql.Tx) error { + _, err := tx.Exec(` +ALTER TABLE device_devices RENAME TO device_devices_tmp; +CREATE TABLE IF NOT EXISTS device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + UNIQUE (localpart, device_id) +); +INSERT +INTO device_devices ( + access_token, session_id, device_id, localpart, created_ts, display_name +) SELECT + access_token, session_id, device_id, localpart, created_ts, display_name +FROM device_devices_tmp; +DROP TABLE device_devices_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql deleted file mode 100644 index 887f90e0..00000000 --- a/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql +++ /dev/null @@ -1,44 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE device_devices RENAME TO device_devices_tmp; -CREATE TABLE device_devices ( - access_token TEXT PRIMARY KEY, - session_id INTEGER, - device_id TEXT , - localpart TEXT , - created_ts BIGINT, - display_name TEXT, - last_seen_ts BIGINT, - ip TEXT, - user_agent TEXT, - UNIQUE (localpart, device_id) -); -INSERT -INTO device_devices ( - access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent -) SELECT - access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', '' -FROM device_devices_tmp; -DROP TABLE device_devices_tmp; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE device_devices RENAME TO device_devices_tmp; -CREATE TABLE IF NOT EXISTS device_devices ( - access_token TEXT PRIMARY KEY, - session_id INTEGER, - device_id TEXT , - localpart TEXT , - created_ts BIGINT, - display_name TEXT, - UNIQUE (localpart, device_id) -); -INSERT -INTO device_devices ( - access_token, session_id, device_id, localpart, created_ts, display_name -) SELECT - access_token, session_id, device_id, localpart, created_ts, display_name -FROM device_devices_tmp; -DROP TABLE device_devices_tmp; --- +goose StatementEnd \ No newline at end of file diff --git a/userapi/storage/devices/sqlite3/devices_table.go b/userapi/storage/devices/sqlite3/devices_table.go index 6b0de10e..26c03222 100644 --- a/userapi/storage/devices/sqlite3/devices_table.go +++ b/userapi/storage/devices/sqlite3/devices_table.go @@ -98,13 +98,14 @@ type devicesStatements struct { serverName gomatrixserverlib.ServerName } +func (s *devicesStatements) execSchema(db *sql.DB) error { + _, err := db.Exec(devicesSchema) + return err +} + func (s *devicesStatements) prepare(db *sql.DB, writer sqlutil.Writer, server gomatrixserverlib.ServerName) (err error) { s.db = db s.writer = writer - _, err = db.Exec(devicesSchema) - if err != nil { - return - } if s.insertDeviceStmt, err = db.Prepare(insertDeviceSQL); err != nil { return } diff --git a/userapi/storage/devices/sqlite3/storage.go b/userapi/storage/devices/sqlite3/storage.go index cfaf4fd9..25888eae 100644 --- a/userapi/storage/devices/sqlite3/storage.go +++ b/userapi/storage/devices/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3/deltas" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -46,6 +47,17 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver } writer := sqlutil.NewExclusiveWriter() d := devicesStatements{} + + // Create tables before executing migrations so we don't fail if the table is missing, + // and THEN prepare statements so we don't fail due to referencing new columns + if err = d.execSchema(db); err != nil { + return nil, err + } + m := sqlutil.NewMigrations() + deltas.LoadLastSeenTSIP(m) + if err = m.RunDeltas(db, dbProperties); err != nil { + return nil, err + } if err = d.prepare(db, writer, serverName); err != nil { return nil, err }