From a97b8eafd459c6a5c742333e2d0d93385da6db95 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 6 Mar 2020 10:23:55 +0000 Subject: [PATCH] Add peer-to-peer support into Dendrite via libp2p and fetch (#880) * Use a fork of pq which supports userCurrent on wasm * Use sqlite3_js driver when running in JS * Add cmd/dendritejs to pull in sqlite3_js driver for wasm only * Update to latest go-sqlite-js version * Replace prometheus with a stub. sigh * Hard-code a config and don't use opentracing * Latest go-sqlite3-js version * Generate a key for now * Listen for fetch traffic rather than HTTP * Latest hacks for js * libp2p support * More libp2p * Fork gjson to allow us to enforce auth checks as before Previously, all events would come down redacted because the hash checks would fail. They would fail because sjson.DeleteBytes didn't remove keys not used for hashing. This didn't work because of a build tag which included a file which no-oped the index returned. See https://github.com/tidwall/gjson/issues/157 When it's resolved, let's go back to mainline. * Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157 * Use latest gomatrixserverlib for sig checks * Fix a bug which could cause exclude_from_sync to not be set Caused when sending events over federation. * Use query variadic to make lookups actually work! * Latest gomatrixserverlib * Add notes on getting p2p up and running Partly so I don't forget myself! * refactor: Move p2p specific stuff to cmd/dendritejs This is important or else the normal build of dendrite will fail because the p2p libraries depend on syscall/js which doesn't work on normal builds. Also, clean up main.go to read a bit better. * Update ho-http-js-libp2p to return errors from RoundTrip * Add an LRU cache around the key DB We actually need this for P2P because otherwise we can *segfault* with things like: "runtime: unexpected return pc for runtime.handleEvent" where the event is a `syscall/js` event, caused by spamming sql.js caused by "Checking event signatures for 14 events of room state" which hammers the key DB repeatedly in quick succession. Using a cache fixes this, though the underlying cause is probably a bug in the version of Go I'm on (1.13.7) * breaking: Add Tracing.Enabled to toggle whether we do opentracing Defaults to false, which is why this is a breaking change. We need this flag because WASM builds cannot do opentracing. * Start adding conditional builds for wasm to handle lib/pq The general idea here is to have the wasm build have a `NewXXXDatabase` that doesn't import any postgres package and hence we never import `lib/pq`, which doesn't work under WASM (undefined `userCurrent`). * Remove lib/pq for wasm for syncapi * Add conditional building to remaining storage APIs * Update build script to set env vars correctly for dendritejs * sqlite bug fixes * Docs * Add a no-op main for dendritejs when not building under wasm * Use the real prometheus, even for WASM Instead, the dendrite-sw.js must mock out `process.pid` and `fs.stat` - which must invoke the callback with an error (e.g `EINVAL`) in order for it to work: ``` global.process = { pid: 1, }; global.fs.stat = function(path, cb) { cb({ code: "EINVAL", }); } ``` * Linting --- appservice/storage/interface.go | 30 ++++ appservice/storage/sqlite3/storage.go | 3 +- appservice/storage/storage.go | 13 +- appservice/storage/storage_wasm.go | 37 ++++ build.sh | 7 +- clientapi/auth/storage/accounts/interface.go | 52 ++++++ .../accounts/sqlite3/membership_table.go | 15 +- .../auth/storage/accounts/sqlite3/storage.go | 2 +- clientapi/auth/storage/accounts/storage.go | 48 ++---- .../auth/storage/accounts/storage_wasm.go | 38 +++++ clientapi/auth/storage/devices/interface.go | 32 ++++ .../auth/storage/devices/sqlite3/storage.go | 2 +- clientapi/auth/storage/devices/storage.go | 29 ++-- .../auth/storage/devices/storage_wasm.go | 38 +++++ clientapi/routing/register.go | 1 - cmd/dendritejs/main.go | 160 ++++++++++++++++++ cmd/dendritejs/main_noop.go | 23 +++ common/basecomponent/base.go | 2 +- common/config/config.go | 17 +- common/keydb/interface.go | 13 ++ common/keydb/keydb.go | 9 +- common/keydb/keydb_wasm.go | 46 +++++ common/keydb/sqlite3/keydb.go | 3 +- common/keydb/sqlite3/server_key_table.go | 40 ++++- common/postgres.go | 25 +++ common/postgres_wasm.go | 22 +++ common/sql.go | 16 +- federationsender/storage/interface.go | 28 +++ federationsender/storage/sqlite3/storage.go | 2 +- federationsender/storage/storage.go | 11 +- federationsender/storage/storage_wasm.go | 38 +++++ go.mod | 37 ++-- go.sum | 63 +++++-- mediaapi/storage/interface.go | 30 ++++ mediaapi/storage/sqlite3/storage.go | 3 +- mediaapi/storage/storage.go | 13 +- mediaapi/storage/storage_wasm.go | 38 +++++ p2p.md | 81 +++++++++ publicroomsapi/storage/interface.go | 33 ++++ .../storage/sqlite3/public_rooms_table.go | 15 +- publicroomsapi/storage/sqlite3/storage.go | 2 +- publicroomsapi/storage/storage.go | 16 +- publicroomsapi/storage/storage_wasm.go | 38 +++++ roomserver/storage/interface.go | 49 ++++++ roomserver/storage/sqlite3/events_table.go | 15 +- roomserver/storage/sqlite3/list.go | 18 -- roomserver/storage/sqlite3/rooms_table.go | 24 +-- .../storage/sqlite3/state_block_table.go | 7 +- .../storage/sqlite3/state_snapshot_table.go | 21 ++- roomserver/storage/sqlite3/storage.go | 2 +- roomserver/storage/storage.go | 32 +--- roomserver/storage/storage_wasm.go | 38 +++++ syncapi/storage/interface.go | 53 ++++++ .../sqlite3/current_room_state_table.go | 9 +- syncapi/storage/sqlite3/filtering.go | 36 ---- .../sqlite3/output_room_events_table.go | 53 ++++-- syncapi/storage/sqlite3/syncserver.go | 5 +- syncapi/storage/storage.go | 36 +--- syncapi/storage/storage_wasm.go | 38 +++++ 59 files changed, 1260 insertions(+), 347 deletions(-) create mode 100644 appservice/storage/interface.go create mode 100644 appservice/storage/storage_wasm.go create mode 100644 clientapi/auth/storage/accounts/interface.go create mode 100644 clientapi/auth/storage/accounts/storage_wasm.go create mode 100644 clientapi/auth/storage/devices/interface.go create mode 100644 clientapi/auth/storage/devices/storage_wasm.go create mode 100644 cmd/dendritejs/main.go create mode 100644 cmd/dendritejs/main_noop.go create mode 100644 common/keydb/interface.go create mode 100644 common/keydb/keydb_wasm.go create mode 100644 common/postgres.go create mode 100644 common/postgres_wasm.go create mode 100644 federationsender/storage/interface.go create mode 100644 federationsender/storage/storage_wasm.go create mode 100644 mediaapi/storage/interface.go create mode 100644 mediaapi/storage/storage_wasm.go create mode 100644 p2p.md create mode 100644 publicroomsapi/storage/interface.go create mode 100644 publicroomsapi/storage/storage_wasm.go create mode 100644 roomserver/storage/interface.go delete mode 100644 roomserver/storage/sqlite3/list.go create mode 100644 roomserver/storage/storage_wasm.go create mode 100644 syncapi/storage/interface.go delete mode 100644 syncapi/storage/sqlite3/filtering.go create mode 100644 syncapi/storage/storage_wasm.go diff --git a/appservice/storage/interface.go b/appservice/storage/interface.go new file mode 100644 index 00000000..4b75ff68 --- /dev/null +++ b/appservice/storage/interface.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.Event) error + GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.Event, bool, error) + CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) + UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error + RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error + GetLatestTxnID(ctx context.Context) (int, error) +} diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 56ab55f2..5040b61b 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" // Import SQLite database driver + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" ) @@ -35,7 +36,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index ce3776bc..9fbd2a1f 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -12,26 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.Event) error - GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.Event, bool, error) - CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) - UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error - RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error - GetLatestTxnID(ctx context.Context) (int, error) -} - func NewDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go new file mode 100644 index 00000000..2bd1433f --- /dev/null +++ b/appservice/storage/storage_wasm.go @@ -0,0 +1,37 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/appservice/storage/sqlite3" +) + +func NewDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/build.sh b/build.sh index cb109111..3ef14889 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,6 @@ -#!/bin/sh +#!/bin/bash -eu -GOBIN=$PWD/`dirname $0`/bin go install -v $PWD/`dirname $0`/cmd/... +# Put installed packages into ./bin +export GOBIN=$PWD/`dirname $0`/bin + +go install -v $PWD/`dirname $0`/cmd/... \ No newline at end of file diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go new file mode 100644 index 00000000..83d3ee72 --- /dev/null +++ b/clientapi/auth/storage/accounts/interface.go @@ -0,0 +1,52 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package accounts + +import ( + "context" + "errors" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) + GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) + SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error + SetDisplayName(ctx context.Context, localpart string, displayName string) error + CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) + UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error + GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) + GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) + SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error + GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) + GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) + GetNewNumericLocalpart(ctx context.Context) (int64, error) + SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) + RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) + GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) + GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) + GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) + PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) + CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) + GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) +} + +// Err3PIDInUse is the error returned when trying to save an association involving +// a third-party identifier which is already associated to a local user. +var Err3PIDInUse = errors.New("This third-party identifier is already in use") diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 8e5e69ba..38f21b7f 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -17,9 +17,10 @@ package sqlite3 import ( "context" "database/sql" + "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" ) const membershipSchema = ` @@ -54,7 +55,6 @@ const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id IN ($1)" type membershipStatements struct { - deleteMembershipsByEventIDsStmt *sql.Stmt insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt @@ -65,9 +65,6 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.deleteMembershipsByEventIDsStmt, err = db.Prepare(deleteMembershipsByEventIDsSQL); err != nil { - return - } if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil { return } @@ -91,8 +88,12 @@ func (s *membershipStatements) insertMembership( func (s *membershipStatements) deleteMembershipsByEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) (err error) { - stmt := txn.Stmt(s.deleteMembershipsByEventIDsStmt) - _, err = stmt.ExecContext(ctx, pq.StringArray(eventIDs)) + sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", common.QueryVariadic(len(eventIDs)), 1) + iEventIDs := make([]interface{}, len(eventIDs)) + for i, e := range eventIDs { + iEventIDs[i] = e + } + _, err = txn.ExecContext(ctx, sqlStr, iEventIDs...) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 199c4606..3e62d10d 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -45,7 +45,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 1dfd5f1f..c643a4d0 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -1,41 +1,29 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + package accounts import ( - "context" - "errors" "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) - GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) - SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error - SetDisplayName(ctx context.Context, localpart string, displayName string) error - CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) - UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error - GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) - GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) - SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error - GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) - GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) - GetNewNumericLocalpart(ctx context.Context) (int64, error) - SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) - RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) - GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) - GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) - GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) - PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) - CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) - GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) -} - func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { @@ -50,7 +38,3 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) return postgres.NewDatabase(dataSourceName, serverName) } } - -// Err3PIDInUse is the error returned when trying to save an association involving -// a third-party identifier which is already associated to a local user. -var Err3PIDInUse = errors.New("This third-party identifier is already in use") diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go new file mode 100644 index 00000000..828afc6b --- /dev/null +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package accounts + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/clientapi/auth/storage/devices/interface.go b/clientapi/auth/storage/devices/interface.go new file mode 100644 index 00000000..95291e4a --- /dev/null +++ b/clientapi/auth/storage/devices/interface.go @@ -0,0 +1,32 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devices + +import ( + "context" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" +) + +type Database interface { + GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) + GetDeviceByID(ctx context.Context, localpart, deviceID string) (*authtypes.Device, error) + GetDevicesByLocalpart(ctx context.Context, localpart string) ([]authtypes.Device, error) + CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *authtypes.Device, returnErr error) + UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error + RemoveDevice(ctx context.Context, deviceID, localpart string) error + RemoveDevices(ctx context.Context, localpart string, devices []string) error + RemoveAllDevices(ctx context.Context, localpart string) error +} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index e1ce6f00..62c30322 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -40,7 +40,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 82f75640..99211db2 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -1,26 +1,29 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + package devices import ( - "context" "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) - GetDeviceByID(ctx context.Context, localpart, deviceID string) (*authtypes.Device, error) - GetDevicesByLocalpart(ctx context.Context, localpart string) ([]authtypes.Device, error) - CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *authtypes.Device, returnErr error) - UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error - RemoveDevice(ctx context.Context, deviceID, localpart string) error - RemoveDevices(ctx context.Context, localpart string, devices []string) error - RemoveAllDevices(ctx context.Context, localpart string) error -} - func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go new file mode 100644 index 00000000..32285288 --- /dev/null +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devices + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 6df0b28b..ba24e527 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -444,7 +444,6 @@ func Register( deviceDB devices.Database, cfg *config.Dendrite, ) util.JSONResponse { - var r registerRequest resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go new file mode 100644 index 00000000..347a5283 --- /dev/null +++ b/cmd/dendritejs/main.go @@ -0,0 +1,160 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "crypto/ed25519" + "fmt" + "net/http" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/mediaapi" + "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/syncapi" + "github.com/matrix-org/dendrite/typingserver" + "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/sirupsen/logrus" + + _ "github.com/matrix-org/go-sqlite3-js" +) + +func init() { + fmt.Println("dendrite.js starting...") +} + +func generateKey() ed25519.PrivateKey { + _, priv, err := ed25519.GenerateKey(nil) + if err != nil { + logrus.Fatalf("Failed to generate ed25519 key: %s", err) + } + return priv +} + +func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.FederationClient { + fmt.Println("Running in js-libp2p federation mode") + fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!") + tr := go_http_js_libp2p.NewP2pTransport(node) + + fed := gomatrixserverlib.NewFederationClient( + cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, + ) + fed.Client = *gomatrixserverlib.NewClientWithTransport(tr) + + return fed +} + +func createP2PNode() (serverName string, node *go_http_js_libp2p.P2pLocalNode) { + hosted := "/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star" + _ = "/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/" + node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", []string{hosted}) + serverName = node.Id + fmt.Println("p2p assigned ServerName: ", serverName) + return +} + +func main() { + cfg := &config.Dendrite{} + cfg.SetDefaults() + cfg.Kafka.UseNaffka = true + cfg.Database.Account = "file:dendritejs_account.db" + cfg.Database.AppService = "file:dendritejs_appservice.db" + cfg.Database.Device = "file:dendritejs_device.db" + cfg.Database.FederationSender = "file:dendritejs_fedsender.db" + cfg.Database.MediaAPI = "file:dendritejs_mediaapi.db" + cfg.Database.Naffka = "file:dendritejs_naffka.db" + cfg.Database.PublicRoomsAPI = "file:dendritejs_publicrooms.db" + cfg.Database.RoomServer = "file:dendritejs_roomserver.db" + cfg.Database.ServerKey = "file:dendritejs_serverkey.db" + cfg.Database.SyncAPI = "file:dendritejs_syncapi.db" + cfg.Matrix.TrustedIDServers = []string{ + "matrix.org", "vector.im", + } + cfg.Matrix.KeyID = "ed25519:1337" + cfg.Matrix.PrivateKey = generateKey() + + serverName, node := createP2PNode() + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(serverName) + + if err := cfg.Derive(); err != nil { + logrus.Fatalf("Failed to derive values from config: %s", err) + } + base := basecomponent.NewBaseDendrite(cfg, "Monolith") + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + keyDB := base.CreateKeyDB() + federation := createFederationClient(cfg, node) + keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + + alias, input, query := roomserver.SetupRoomServerComponent(base) + typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) + asQuery := appservice.SetupAppServiceAPIComponent( + base, accountDB, deviceDB, federation, alias, query, transactions.New(), + ) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + + clientapi.SetupClientAPIComponent( + base, deviceDB, accountDB, + federation, &keyRing, alias, input, query, + typingInputAPI, asQuery, transactions.New(), fedSenderAPI, + ) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) + mediaapi.SetupMediaAPIComponent(base, deviceDB) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + + httpHandler := common.WrapHandlerInCORS(base.APIMux) + + http.Handle("/", httpHandler) + + // Expose the matrix APIs via libp2p-js - for federation traffic + if node != nil { + go func() { + logrus.Info("Listening on libp2p-js host ID ", node.Id) + + listener := go_http_js_libp2p.NewP2pListener(node) + defer listener.Close() + s := &http.Server{} + s.Serve(listener) + }() + } + + // Expose the matrix APIs via fetch - for local traffic + go func() { + logrus.Info("Listening for service-worker fetch traffic") + + listener := go_http_js_libp2p.NewFetchListener() + s := &http.Server{} + go s.Serve(listener) + }() + + // We want to block forever to let the fetch and libp2p handler serve the APIs + select {} +} diff --git a/cmd/dendritejs/main_noop.go b/cmd/dendritejs/main_noop.go new file mode 100644 index 00000000..dcea032f --- /dev/null +++ b/cmd/dendritejs/main_noop.go @@ -0,0 +1,23 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package main + +import "fmt" + +func main() { + fmt.Println("dendritejs: no-op when not compiling for WebAssembly") +} diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 4274de2b..d1d953f7 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -216,7 +216,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sql.Open("sqlite3", string(cfg.Database.Naffka)) + db, err = sql.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/config/config.go b/common/config/config.go index 0332d035..bd83cbf8 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -224,6 +224,8 @@ type Dendrite struct { // The config for tracing the dendrite servers. Tracing struct { + // Set to true to enable tracer hooks. If false, no tracing is set up. + Enabled bool `yaml:"enabled"` // The config for the jaeger opentracing reporter. Jaeger jaegerconfig.Configuration `yaml:"jaeger"` } `yaml:"tracing"` @@ -365,7 +367,7 @@ func loadConfig( return nil, err } - config.setDefaults() + config.SetDefaults() if err = config.check(monolithic); err != nil { return nil, err @@ -398,7 +400,7 @@ func loadConfig( config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) // Generate data from config options - err = config.derive() + err = config.Derive() if err != nil { return nil, err } @@ -406,9 +408,9 @@ func loadConfig( return &config, nil } -// derive generates data that is derived from various values provided in +// Derive generates data that is derived from various values provided in // the config file. -func (config *Dendrite) derive() error { +func (config *Dendrite) Derive() error { // Determine registrations flows based off config values config.Derived.Registration.Params = make(map[string]interface{}) @@ -433,8 +435,8 @@ func (config *Dendrite) derive() error { return nil } -// setDefaults sets default config values if they are not explicitly set. -func (config *Dendrite) setDefaults() { +// SetDefaults sets default config values if they are not explicitly set. +func (config *Dendrite) SetDefaults() { if config.Matrix.KeyValidityPeriod == 0 { config.Matrix.KeyValidityPeriod = 24 * time.Hour } @@ -703,6 +705,9 @@ func (config *Dendrite) FederationSenderURL() string { // SetupTracing configures the opentracing using the supplied configuration. func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { + if !config.Tracing.Enabled { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } return config.Tracing.Jaeger.InitGlobalTracer( serviceName, jaegerconfig.Logger(logrusLogger{logrus.StandardLogger()}), diff --git a/common/keydb/interface.go b/common/keydb/interface.go new file mode 100644 index 00000000..c9a20fdd --- /dev/null +++ b/common/keydb/interface.go @@ -0,0 +1,13 @@ +package keydb + +import ( + "context" + + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + FetcherName() string + FetchKeys(ctx context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) + StoreKeys(ctx context.Context, keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error +} diff --git a/common/keydb/keydb.go b/common/keydb/keydb.go index cf15c9f0..fe6d87fc 100644 --- a/common/keydb/keydb.go +++ b/common/keydb/keydb.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package keydb import ( - "context" "net/url" "golang.org/x/crypto/ed25519" @@ -25,12 +26,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - FetcherName() string - FetchKeys(ctx context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) - StoreKeys(ctx context.Context, keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error -} - // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, diff --git a/common/keydb/keydb_wasm.go b/common/keydb/keydb_wasm.go new file mode 100644 index 00000000..807ed40b --- /dev/null +++ b/common/keydb/keydb_wasm.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keydb + +import ( + "fmt" + "net/url" + + "golang.org/x/crypto/ed25519" + + "github.com/matrix-org/dendrite/common/keydb/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +// NewDatabase opens a database connection. +func NewDatabase( + dataSourceName string, + serverName gomatrixserverlib.ServerName, + serverKey ed25519.PublicKey, + serverKeyID gomatrixserverlib.KeyID, +) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 88eb9d9f..3c860d0c 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -22,6 +22,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -43,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sql.Open("sqlite3", dataSourceName) + db, err := sql.Open(common.SQLiteDriverName(), dataSourceName) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go index 6c33f30a..3caf4ea4 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/common/keydb/sqlite3/server_key_table.go @@ -18,9 +18,12 @@ package sqlite3 import ( "context" "database/sql" + "strings" - "github.com/lib/pq" + lru "github.com/hashicorp/golang-lru" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) const serverKeysSchema = ` @@ -60,11 +63,19 @@ const upsertServerKeysSQL = "" + " DO UPDATE SET valid_until_ts = $4, expired_ts = $5, server_key = $6" type serverKeyStatements struct { + db *sql.DB bulkSelectServerKeysStmt *sql.Stmt upsertServerKeysStmt *sql.Stmt + + cache *lru.Cache // nameAndKeyID => gomatrixserverlib.PublicKeyLookupResult } func (s *serverKeyStatements) prepare(db *sql.DB) (err error) { + s.db = db + s.cache, err = lru.New(64) + if err != nil { + return + } _, err = db.Exec(serverKeysSchema) if err != nil { return @@ -86,8 +97,30 @@ func (s *serverKeyStatements) bulkSelectServerKeys( for request := range requests { nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) } - stmt := s.bulkSelectServerKeysStmt - rows, err := stmt.QueryContext(ctx, pq.StringArray(nameAndKeyIDs)) + + // If we can satisfy all of the requests from the cache, do so. TODO: Allow partial matches with merges. + cacheResults := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + for request := range requests { + r, ok := s.cache.Get(nameAndKeyID(request)) + if !ok { + break + } + cacheResult := r.(gomatrixserverlib.PublicKeyLookupResult) + cacheResults[request] = cacheResult + } + if len(cacheResults) == len(requests) { + util.GetLogger(ctx).Infof("KeyDB cache hit for %d keys", len(cacheResults)) + return cacheResults, nil + } + + query := strings.Replace(bulkSelectServerKeysSQL, "($1)", common.QueryVariadic(len(nameAndKeyIDs)), 1) + + iKeyIDs := make([]interface{}, len(nameAndKeyIDs)) + for i, v := range nameAndKeyIDs { + iKeyIDs[i] = v + } + + rows, err := s.db.QueryContext(ctx, query, iKeyIDs...) if err != nil { return nil, err } @@ -125,6 +158,7 @@ func (s *serverKeyStatements) upsertServerKeys( request gomatrixserverlib.PublicKeyLookupRequest, key gomatrixserverlib.PublicKeyLookupResult, ) error { + s.cache.Add(nameAndKeyID(request), key) _, err := s.upsertServerKeysStmt.ExecContext( ctx, string(request.ServerName), diff --git a/common/postgres.go b/common/postgres.go new file mode 100644 index 00000000..f8daf578 --- /dev/null +++ b/common/postgres.go @@ -0,0 +1,25 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package common + +import "github.com/lib/pq" + +// IsUniqueConstraintViolationErr returns true if the error is a postgresql unique_violation error +func IsUniqueConstraintViolationErr(err error) bool { + pqErr, ok := err.(*pq.Error) + return ok && pqErr.Code == "23505" +} diff --git a/common/postgres_wasm.go b/common/postgres_wasm.go new file mode 100644 index 00000000..dcc07b31 --- /dev/null +++ b/common/postgres_wasm.go @@ -0,0 +1,22 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package common + +// IsUniqueConstraintViolationErr no-ops for this architecture +func IsUniqueConstraintViolationErr(err error) bool { + return false +} diff --git a/common/sql.go b/common/sql.go index 97593020..f50a5896 100644 --- a/common/sql.go +++ b/common/sql.go @@ -17,8 +17,7 @@ package common import ( "database/sql" "fmt" - - "github.com/lib/pq" + "runtime" ) // A Transaction is something that can be committed or rolledback. @@ -77,12 +76,6 @@ func TxStmt(transaction *sql.Tx, statement *sql.Stmt) *sql.Stmt { return statement } -// IsUniqueConstraintViolationErr returns true if the error is a postgresql unique_violation error -func IsUniqueConstraintViolationErr(err error) bool { - pqErr, ok := err.(*pq.Error) - return ok && pqErr.Code == "23505" -} - // Hack of the century func QueryVariadic(count int) string { return QueryVariadicOffset(count, 0) @@ -99,3 +92,10 @@ func QueryVariadicOffset(count, offset int) string { str += ")" return str } + +func SQLiteDriverName() string { + if runtime.GOOS == "js" { + return "sqlite3_js" + } + return "sqlite3" +} diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go new file mode 100644 index 00000000..ae295647 --- /dev/null +++ b/federationsender/storage/interface.go @@ -0,0 +1,28 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/federationsender/types" +) + +type Database interface { + common.PartitionStorer + UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) + GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) +} diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index f9cfaa99..6ab9ae00 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index e83c1e9d..2f018dff 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -12,24 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/federationsender/types" ) -type Database interface { - common.PartitionStorer - UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) - GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) -} - // NewDatabase opens a new database func NewDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go new file mode 100644 index 00000000..f2c8ae1b --- /dev/null +++ b/federationsender/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" +) + +// NewDatabase opens a new database +func NewDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "file": + return sqlite3.NewDatabase(dataSourceName) + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/go.mod b/go.mod index 2d442cd0..7d59aeb1 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,30 @@ module github.com/matrix-org/dendrite require ( - github.com/DataDog/zstd v1.4.4 // indirect - github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect - github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/eapache/go-resiliency v1.2.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect - github.com/eapache/queue v1.1.0 // indirect - github.com/frankban/quicktest v1.7.2 // indirect - github.com/golang/snappy v0.0.1 // indirect github.com/gorilla/mux v1.7.3 - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 + github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 - github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 + github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible - github.com/miekg/dns v1.1.12 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 - github.com/opentracing/opentracing-go v1.0.2 - github.com/pierrec/lz4 v2.4.1+incompatible // indirect + github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 - github.com/prometheus/client_golang v1.2.1 - github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect + github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.4.0 // indirect - github.com/uber-go/atomic v1.3.0 // indirect - github.com/uber/jaeger-client-go v2.15.0+incompatible - github.com/uber/jaeger-lib v1.5.0 - go.uber.org/atomic v1.3.0 // indirect + github.com/tidwall/gjson v1.6.0 + github.com/tidwall/pretty v1.0.1 // indirect + github.com/uber/jaeger-client-go v2.22.1+incompatible + github.com/uber/jaeger-lib v2.2.0+incompatible golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 - golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect - golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect gopkg.in/Shopify/sarama.v1 v1.20.1 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 - gopkg.in/yaml.v2 v2.2.2 + gopkg.in/yaml.v2 v2.2.5 ) go 1.13 diff --git a/go.sum b/go.sum index 7c8732f6..19dc9662 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= @@ -40,17 +41,20 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= @@ -63,18 +67,37 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94 h1:gUqPXKWbuwmPnx3PJBxtQw5Ff8V229wZ7Ee0GaH/bg4= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 h1:oFjG7X1jS473zPDix1/FBZ2qd0anM1Ko4AlJCB6MUZs= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8 h1:a9IV2iKMznKJ16MmcG/NU7qMcZ4jIKmPXC6RkOZZq+Q= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e h1:DA1lP2mB2ddd2PhMOaNPwRJFi/3aL2Lj7bxnAhFxBFQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= +github.com/matrix-org/pq v1.3.2 h1:7Kh2Qz4xonRH8OcFpCIj2najpBxF2+j1ff0hC082L68= +github.com/matrix-org/pq v1.3.2/go.mod h1:l6tPTzDjcj8fhD5OD0+7dejtZrwyjJadAAYM+CntUVQ= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= +github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= +github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -94,6 +117,8 @@ github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/R github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -103,19 +128,23 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= +github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -126,22 +155,30 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= +github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= +github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= +github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= +github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -156,24 +193,26 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -181,6 +220,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= @@ -190,3 +230,6 @@ gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/mediaapi/storage/interface.go b/mediaapi/storage/interface.go new file mode 100644 index 00000000..672e8ef5 --- /dev/null +++ b/mediaapi/storage/interface.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error + GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) + StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error + GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) + GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) +} diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index 6477f830..cfa5bb98 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" // Import the postgres database driver. + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -35,7 +36,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 1d35a95e..c533477c 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -12,26 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" - "github.com/matrix-org/dendrite/mediaapi/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error - GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) - StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error - GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) - GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) -} - // Open opens a postgres database. func Open(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go new file mode 100644 index 00000000..92f0ad13 --- /dev/null +++ b/mediaapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" +) + +// Open opens a postgres database. +func Open(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.Open(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/p2p.md b/p2p.md new file mode 100644 index 00000000..0c3dc83b --- /dev/null +++ b/p2p.md @@ -0,0 +1,81 @@ +## Peer-to-peer Matrix + +These are the instructions for setting up P2P Dendrite, current as of March 2020. There's both Go stuff and JS stuff to do to set this up. + + +### Dendrite + +- The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`. +- Build it and copy assets to riot-web. +``` +$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs +$ cp main.wasm ../riot-web/src/vector/dendrite.wasm +``` + +### Rendezvous + +This is how peers discover each other and communicate. + +By default, Dendrite uses the IPFS-hosted websocket star **Development** relay server at `/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star`. +This is currently hard-coded in `./cmd/dendritejs/main.go` - you can also use a local one if you run your own relay: + +``` +$ npm install --global libp2p-websocket-star-rendezvous +$ rendezvous --port=9090 --host=127.0.0.1 +``` + +Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`. We'll probably run our own relay server at some point. + +### Riot-web + +You need to check out these repos: + +`` +$ git clone git@github.com:matrix-org/go-http-js-libp2p.git +$ git clone git@github.com:matrix-org/go-sqlite3-js.git +``` + +Make sure to `yarn install` in both of these repos. Then: + +- `$ cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" ./src/vector/` +- Comment out the lines in `wasm_exec.js` which contains: +``` +if (!global.fs && global.require) { + global.fs = require("fs"); +} +``` +- Add the diff at https://github.com/vector-im/riot-web/compare/matthew/p2p?expand=1 - ignore the `package.json` stuff. +- Add the following symlinks: they HAVE to be symlinks as the diff in `webpack.config.js` references specific paths. +``` +$ cd node_modules +$ ln -s ../../go-sqlite-js # NB: NOT go-sqlite3-js +$ ln -s ../../go-http-js-libp2p +``` + +NB: If you don't run the server with `yarn start` you need to make sure your server is sending the header `Service-Worker-Allowed: /`. + +TODO: Make a Docker image with all of this in it and a volume mount for `dendrite.wasm`. + +## Running + +You need a Chrome and a Firefox running to test locally as service workers don't work in incognito tabs. +- For Chrome, use `chrome://serviceworker-internals/` to unregister/see logs. +- For Firefox, use `about:debugging#/runtime/this-firefox` to unregister. Use the console window to see logs. + +Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and wait a bit. Then refresh the page (this is required +because the fetch interceptor races with setting up dendrite. If you don't refresh, you won't be able to contact your HS). After +the refresh, click Register and use `http://localhost:8080` as your HS URL. + +TODO: Fix the race so we don't need multiple refreshes. + +You can join rooms by room alias e.g `/join #foo:bar`. + +### Known issues + +- When registering you may be unable to find the server, it'll seem flakey. This happens because the SW, particularly in Firefox, + gets killed after 30s of inactivity. When you are not registered, you aren't doing `/sync` calls to keep the SW alive, so if you + don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page *twice*. + +- The libp2p layer has rate limits, so frequent Federation traffic may cause the connection to drop and messages to not be transferred. + I guess in other words, don't send too much traffic? + diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go new file mode 100644 index 00000000..3f7e6589 --- /dev/null +++ b/publicroomsapi/storage/interface.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + GetRoomVisibility(ctx context.Context, roomID string) (bool, error) + SetRoomVisibility(ctx context.Context, visible bool, roomID string) error + CountPublicRooms(ctx context.Context) (int64, error) + GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) + UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error + UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error +} diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go index 3fe3d3ca..beba0d69 100644 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "errors" "fmt" - "github.com/lib/pq" "github.com/matrix-org/dendrite/publicroomsapi/types" ) @@ -196,17 +196,21 @@ func (s *publicRoomsStatements) selectPublicRooms( rooms := []types.PublicRoom{} for rows.Next() { var r types.PublicRoom - var aliases pq.StringArray + var aliasesJSON string err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias, + &r.RoomID, &r.NumJoinedMembers, &aliasesJSON, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { return rooms, err } - r.Aliases = aliases + if len(aliasesJSON) > 0 { + if err := json.Unmarshal([]byte(aliasesJSON), &r.Aliases); err != nil { + return rooms, err + } + } rooms = append(rooms, r) } @@ -254,7 +258,8 @@ func (s *publicRoomsStatements) updateRoomAttribute( var value interface{} switch v := attrValue.(type) { case []string: - value = pq.StringArray(v) + b, _ := json.Marshal(v) + value = string(b) case bool, string: value = attrValue default: diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index dcb8920f..f8ba71a8 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -41,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index 29a6619f..41dbd781 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -12,29 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/dendrite/publicroomsapi/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - GetRoomVisibility(ctx context.Context, roomID string) (bool, error) - SetRoomVisibility(ctx context.Context, visible bool, roomID string) error - CountPublicRooms(ctx context.Context) (int64, error) - GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) - UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error - UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error -} - // NewPublicRoomsServerDatabase opens a database connection. func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go new file mode 100644 index 00000000..d00c339d --- /dev/null +++ b/publicroomsapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go new file mode 100644 index 00000000..f03d6fea --- /dev/null +++ b/roomserver/storage/interface.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/api" + statedb "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + statedb.RoomStateDatabase + StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) + StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) + EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) + EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) + SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error + EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) + GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) + GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) + RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) + LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) + GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) + SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error + GetRoomIDForAlias(ctx context.Context, alias string) (string, error) + GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) + GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) + RemoveRoomAlias(ctx context.Context, alias string) error + MembershipUpdater(ctx context.Context, roomID, targetUserID string) (types.MembershipUpdater, error) + GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) + GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) + EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) + GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) +} diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 4ed1395d..56e37910 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "fmt" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -38,7 +38,7 @@ const eventsSchema = ` depth INTEGER NOT NULL, event_id TEXT NOT NULL UNIQUE, reference_sha256 BLOB NOT NULL, - auth_event_nids TEXT NOT NULL DEFAULT '{}' + auth_event_nids TEXT NOT NULL DEFAULT '[]' ); ` @@ -463,17 +463,14 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { var result int64 selectStmt := common.TxStmt(txn, s.selectMaxEventDepthStmt) - err := selectStmt.QueryRowContext(ctx, sqliteIn(eventNIDsAsArray(eventNIDs))).Scan(&result) + err := selectStmt.QueryRowContext(ctx, eventNIDsAsArray(eventNIDs)).Scan(&result) if err != nil { return 0, err } return result, nil } -func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { - nids := make([]int64, len(eventNIDs)) - for i := range eventNIDs { - nids[i] = int64(eventNIDs[i]) - } - return nids +func eventNIDsAsArray(eventNIDs []types.EventNID) string { + b, _ := json.Marshal(eventNIDs) + return string(b) } diff --git a/roomserver/storage/sqlite3/list.go b/roomserver/storage/sqlite3/list.go deleted file mode 100644 index 4fe4e334..00000000 --- a/roomserver/storage/sqlite3/list.go +++ /dev/null @@ -1,18 +0,0 @@ -package sqlite3 - -import ( - "strconv" - "strings" - - "github.com/lib/pq" -) - -type SqliteList string - -func sqliteIn(a pq.Int64Array) string { - var b []string - for _, n := range a { - b = append(b, strconv.FormatInt(n, 10)) - } - return strings.Join(b, ",") -} diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index bf237728..99c03476 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -18,8 +18,8 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -28,7 +28,7 @@ const roomsSchema = ` CREATE TABLE IF NOT EXISTS roomserver_rooms ( room_nid INTEGER PRIMARY KEY AUTOINCREMENT, room_id TEXT NOT NULL UNIQUE, - latest_event_nids TEXT NOT NULL DEFAULT '{}', + latest_event_nids TEXT NOT NULL DEFAULT '[]', last_event_sent_nid INTEGER NOT NULL DEFAULT 0, state_snapshot_nid INTEGER NOT NULL DEFAULT 0, room_version INTEGER NOT NULL DEFAULT 1 @@ -104,16 +104,16 @@ func (s *roomStatements) selectRoomNID( func (s *roomStatements) selectLatestEventNIDs( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.StateSnapshotNID, error) { - var nids pq.Int64Array + var eventNIDs []types.EventNID + var nidsJSON string var stateSnapshotNID int64 stmt := common.TxStmt(txn, s.selectLatestEventNIDsStmt) - err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &stateSnapshotNID) + err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &stateSnapshotNID) if err != nil { return nil, 0, err } - eventNIDs := make([]types.EventNID, len(nids)) - for i := range nids { - eventNIDs[i] = types.EventNID(nids[i]) + if err := json.Unmarshal([]byte(nidsJSON), &eventNIDs); err != nil { + return nil, 0, err } return eventNIDs, types.StateSnapshotNID(stateSnapshotNID), nil } @@ -121,17 +121,17 @@ func (s *roomStatements) selectLatestEventNIDs( func (s *roomStatements) selectLatestEventsNIDsForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.EventNID, types.StateSnapshotNID, error) { - var nids pq.Int64Array + var eventNIDs []types.EventNID + var nidsJSON string var lastEventSentNID int64 var stateSnapshotNID int64 stmt := common.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) - err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &lastEventSentNID, &stateSnapshotNID) + err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err } - eventNIDs := make([]types.EventNID, len(nids)) - for i := range nids { - eventNIDs[i] = types.EventNID(nids[i]) + if err := json.Unmarshal([]byte(nidsJSON), &eventNIDs); err != nil { + return nil, 0, 0, err } return eventNIDs, types.EventNID(lastEventSentNID), types.StateSnapshotNID(stateSnapshotNID), nil } diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index d75abcee..c94ca238 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -22,7 +22,6 @@ import ( "sort" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -268,9 +267,9 @@ func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool { // List the unique eventTypeNIDs and eventStateKeyNIDs. // Assumes that the list is sorted. -func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs pq.Int64Array, eventStateKeyNIDs pq.Int64Array) { - eventTypeNIDs = make(pq.Int64Array, len(s)) - eventStateKeyNIDs = make(pq.Int64Array, len(s)) +func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) { + eventTypeNIDs = make([]int64, len(s)) + eventStateKeyNIDs = make([]int64, len(s)) for i := range s { eventTypeNIDs[i] = int64(s[i].EventTypeNID) eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID) diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index df97aa41..0e1786ea 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "fmt" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -30,7 +30,7 @@ const stateSnapshotSchema = ` CREATE TABLE IF NOT EXISTS roomserver_state_snapshots ( state_snapshot_nid INTEGER PRIMARY KEY AUTOINCREMENT, room_nid INTEGER NOT NULL, - state_block_nids TEXT NOT NULL DEFAULT '{}' + state_block_nids TEXT NOT NULL DEFAULT '[]' ); ` @@ -67,12 +67,12 @@ func (s *stateSnapshotStatements) prepare(db *sql.DB) (err error) { func (s *stateSnapshotStatements) insertState( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, ) (stateNID types.StateSnapshotNID, err error) { - nids := make([]int64, len(stateBlockNIDs)) - for i := range stateBlockNIDs { - nids[i] = int64(stateBlockNIDs[i]) + stateBlockNIDsJSON, err := json.Marshal(stateBlockNIDs) + if err != nil { + return } insertStmt := txn.Stmt(s.insertStateStmt) - if res, err2 := insertStmt.ExecContext(ctx, int64(roomNID), pq.Int64Array(nids)); err2 == nil { + if res, err2 := insertStmt.ExecContext(ctx, int64(roomNID), string(stateBlockNIDsJSON)); err2 == nil { lastRowID, err3 := res.LastInsertId() if err3 != nil { err = err3 @@ -104,13 +104,12 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( i := 0 for ; rows.Next(); i++ { result := &results[i] - var stateBlockNIDs pq.Int64Array - if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil { + var stateBlockNIDsJSON string + if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { return nil, err } - result.StateBlockNIDs = make([]types.StateBlockNID, len(stateBlockNIDs)) - for k := range stateBlockNIDs { - result.StateBlockNIDs[k] = types.StateBlockNID(stateBlockNIDs[k]) + if err := json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { + return nil, err } } if i != len(stateNIDs) { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index aebb308c..41d4c2a9 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -49,7 +49,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open("sqlite3", cs); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 551d97cd..7b9109aa 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -12,45 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/roomserver/api" - statedb "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - statedb.RoomStateDatabase - StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) - StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) - EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) - EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) - SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error - EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) - GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) - GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) - RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) - LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) - GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) - SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error - GetRoomIDForAlias(ctx context.Context, alias string) (string, error) - GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) - GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) - RemoveRoomAlias(ctx context.Context, alias string) error - MembershipUpdater(ctx context.Context, roomID, targetUserID string) (types.MembershipUpdater, error) - GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) - GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) - EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) - GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) -} - // NewPublicRoomsServerDatabase opens a database connection. func Open(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go new file mode 100644 index 00000000..d7fc352e --- /dev/null +++ b/roomserver/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func Open(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.Open(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go new file mode 100644 index 00000000..0abeac34 --- /dev/null +++ b/syncapi/storage/interface.go @@ -0,0 +1,53 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) + Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) + WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) + SyncPosition(ctx context.Context) (types.PaginationToken, error) + IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) + GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) + AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) + RetireInviteEvent(ctx context.Context, inviteEventID string) error + SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) + AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition + RemoveTypingUser(userID, roomID string) types.StreamPosition + GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) + EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) + BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) + MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event + SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) +} diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index eb969c95..ed76177b 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -21,7 +21,6 @@ import ( "encoding/json" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -175,10 +174,10 @@ func (s *currentRoomStateStatements) selectCurrentState( ) ([]gomatrixserverlib.Event, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, - pq.StringArray(stateFilterPart.Senders), - pq.StringArray(stateFilterPart.NotSenders), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + nil, // FIXME: pq.StringArray(stateFilterPart.Senders), + nil, // FIXME: pq.StringArray(stateFilterPart.NotSenders), + nil, // FIXME: pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + nil, // FIXME: pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), stateFilterPart.ContainsURL, stateFilterPart.Limit, ) diff --git a/syncapi/storage/sqlite3/filtering.go b/syncapi/storage/sqlite3/filtering.go deleted file mode 100644 index c4a2f4bf..00000000 --- a/syncapi/storage/sqlite3/filtering.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Thibaut CHARLES -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "strings" -) - -// filterConvertWildcardToSQL converts wildcards as defined in -// https://matrix.org/docs/spec/client_server/r0.3.0.html#post-matrix-client-r0-user-userid-filter -// to SQL wildcards that can be used with LIKE() -func filterConvertTypeWildcardToSQL(values []string) []string { - if values == nil { - // Return nil instead of []string{} so IS NULL can work correctly when - // the return value is passed into SQL queries - return nil - } - - ret := make([]string, len(values)) - for i := range values { - ret[i] = strings.Replace(values[i], "*", "%", -1) - } - return ret -} diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 4535688d..be893743 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" @@ -40,11 +39,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( type TEXT NOT NULL, sender TEXT NOT NULL, contains_url BOOL NOT NULL, - add_state_ids TEXT[], - remove_state_ids TEXT[], + add_state_ids TEXT, -- JSON encoded string array + remove_state_ids TEXT, -- JSON encoded string array session_id BIGINT, transaction_id TEXT, - exclude_from_sync BOOL DEFAULT FALSE + exclude_from_sync BOOL NOT NULL DEFAULT FALSE ); ` @@ -52,7 +51,7 @@ const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + "id, room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + - "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $11" + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $13" const selectEventsSQL = "" + "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" @@ -176,20 +175,26 @@ func (s *outputRoomEventsStatements) selectStateInRange( streamPos types.StreamPosition eventBytes []byte excludeFromSync bool - addIDs pq.StringArray - delIDs pq.StringArray + addIDsJSON string + delIDsJSON string ) - if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { + if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil { return nil, nil, err } + + addIDs, delIDs, err := unmarshalStateIDs(addIDsJSON, delIDsJSON) + if err != nil { + return nil, nil, err + } + // Sanity check for deleted state and whine if we see it. We don't need to do anything // since it'll just mark the event as not being needed. if len(addIDs) < len(delIDs) { log.WithFields(log.Fields{ "since": oldPos, "current": newPos, - "adds": addIDs, - "dels": delIDs, + "adds": addIDsJSON, + "dels": delIDsJSON, }).Warn("StateBetween: ignoring deleted state") } @@ -262,6 +267,15 @@ func (s *outputRoomEventsStatements) insertEvent( return } + addStateJSON, err := json.Marshal(addState) + if err != nil { + return + } + removeStateJSON, err := json.Marshal(removeState) + if err != nil { + return + } + insertStmt := common.TxStmt(txn, s.insertEventStmt) _, err = insertStmt.ExecContext( ctx, @@ -272,11 +286,12 @@ func (s *outputRoomEventsStatements) insertEvent( event.Type(), event.Sender(), containsURL, - pq.StringArray(addState), - pq.StringArray(removeState), + string(addStateJSON), + string(removeStateJSON), sessionID, txnID, excludeFromSync, + excludeFromSync, ) return } @@ -397,3 +412,17 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } return result, nil } + +func unmarshalStateIDs(addIDsJSON, delIDsJSON string) (addIDs []string, delIDs []string, err error) { + if len(addIDsJSON) > 0 { + if err = json.Unmarshal([]byte(addIDsJSON), &addIDs); err != nil { + return + } + } + if len(delIDsJSON) > 0 { + if err = json.Unmarshal([]byte(delIDsJSON), &delIDs); err != nil { + return + } + } + return +} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index da580e3a..0e84c8c8 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -29,8 +29,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/roomserver/api" - // Import the postgres database driver. - _ "github.com/lib/pq" + // Import the sqlite3 package _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" @@ -79,7 +78,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open("sqlite3", cs); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } if err = d.prepare(); err != nil { diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index c87024b2..c56db063 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -12,49 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" - "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/typingserver/cache" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) - Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) - WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) - GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) - SyncPosition(ctx context.Context) (types.PaginationToken, error) - IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) - CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) - GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) - UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) - AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) - RetireInviteEvent(ctx context.Context, inviteEventID string) error - SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) - AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition - RemoveTypingUser(userID, roomID string) types.StreamPosition - GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) - EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) - EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) - BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) - MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) - StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event - SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) -} - // NewPublicRoomsServerDatabase opens a database connection. func NewSyncServerDatasource(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go new file mode 100644 index 00000000..43806a01 --- /dev/null +++ b/syncapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func NewSyncServerDatasource(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewSyncServerDatasource(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +}