diff --git a/appservice/storage/interface.go b/appservice/storage/interface.go new file mode 100644 index 00000000..4b75ff68 --- /dev/null +++ b/appservice/storage/interface.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.Event) error + GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.Event, bool, error) + CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) + UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error + RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error + GetLatestTxnID(ctx context.Context) (int, error) +} diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 56ab55f2..5040b61b 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" // Import SQLite database driver + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" ) @@ -35,7 +36,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index ce3776bc..9fbd2a1f 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -12,26 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.Event) error - GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.Event, bool, error) - CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) - UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error - RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error - GetLatestTxnID(ctx context.Context) (int, error) -} - func NewDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go new file mode 100644 index 00000000..2bd1433f --- /dev/null +++ b/appservice/storage/storage_wasm.go @@ -0,0 +1,37 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/appservice/storage/sqlite3" +) + +func NewDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/build.sh b/build.sh index cb109111..3ef14889 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,6 @@ -#!/bin/sh +#!/bin/bash -eu -GOBIN=$PWD/`dirname $0`/bin go install -v $PWD/`dirname $0`/cmd/... +# Put installed packages into ./bin +export GOBIN=$PWD/`dirname $0`/bin + +go install -v $PWD/`dirname $0`/cmd/... \ No newline at end of file diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go new file mode 100644 index 00000000..83d3ee72 --- /dev/null +++ b/clientapi/auth/storage/accounts/interface.go @@ -0,0 +1,52 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package accounts + +import ( + "context" + "errors" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) + GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) + SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error + SetDisplayName(ctx context.Context, localpart string, displayName string) error + CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) + UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error + GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) + GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) + SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error + GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) + GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) + GetNewNumericLocalpart(ctx context.Context) (int64, error) + SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) + RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) + GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) + GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) + GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) + PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) + CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) + GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) +} + +// Err3PIDInUse is the error returned when trying to save an association involving +// a third-party identifier which is already associated to a local user. +var Err3PIDInUse = errors.New("This third-party identifier is already in use") diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 8e5e69ba..38f21b7f 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -17,9 +17,10 @@ package sqlite3 import ( "context" "database/sql" + "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" ) const membershipSchema = ` @@ -54,7 +55,6 @@ const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id IN ($1)" type membershipStatements struct { - deleteMembershipsByEventIDsStmt *sql.Stmt insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt @@ -65,9 +65,6 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.deleteMembershipsByEventIDsStmt, err = db.Prepare(deleteMembershipsByEventIDsSQL); err != nil { - return - } if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil { return } @@ -91,8 +88,12 @@ func (s *membershipStatements) insertMembership( func (s *membershipStatements) deleteMembershipsByEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) (err error) { - stmt := txn.Stmt(s.deleteMembershipsByEventIDsStmt) - _, err = stmt.ExecContext(ctx, pq.StringArray(eventIDs)) + sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", common.QueryVariadic(len(eventIDs)), 1) + iEventIDs := make([]interface{}, len(eventIDs)) + for i, e := range eventIDs { + iEventIDs[i] = e + } + _, err = txn.ExecContext(ctx, sqlStr, iEventIDs...) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 199c4606..3e62d10d 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -45,7 +45,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 1dfd5f1f..c643a4d0 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -1,41 +1,29 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + package accounts import ( - "context" - "errors" "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) - GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) - SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error - SetDisplayName(ctx context.Context, localpart string, displayName string) error - CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) - UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error - GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) - GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) - SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error - GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) - GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) - GetNewNumericLocalpart(ctx context.Context) (int64, error) - SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) - RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) - GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) - GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) - GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) - PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) - CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) - GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) -} - func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { @@ -50,7 +38,3 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) return postgres.NewDatabase(dataSourceName, serverName) } } - -// Err3PIDInUse is the error returned when trying to save an association involving -// a third-party identifier which is already associated to a local user. -var Err3PIDInUse = errors.New("This third-party identifier is already in use") diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go new file mode 100644 index 00000000..828afc6b --- /dev/null +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package accounts + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/clientapi/auth/storage/devices/interface.go b/clientapi/auth/storage/devices/interface.go new file mode 100644 index 00000000..95291e4a --- /dev/null +++ b/clientapi/auth/storage/devices/interface.go @@ -0,0 +1,32 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devices + +import ( + "context" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" +) + +type Database interface { + GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) + GetDeviceByID(ctx context.Context, localpart, deviceID string) (*authtypes.Device, error) + GetDevicesByLocalpart(ctx context.Context, localpart string) ([]authtypes.Device, error) + CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *authtypes.Device, returnErr error) + UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error + RemoveDevice(ctx context.Context, deviceID, localpart string) error + RemoveDevices(ctx context.Context, localpart string, devices []string) error + RemoveAllDevices(ctx context.Context, localpart string) error +} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index e1ce6f00..62c30322 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -40,7 +40,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 82f75640..99211db2 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -1,26 +1,29 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + package devices import ( - "context" "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) - GetDeviceByID(ctx context.Context, localpart, deviceID string) (*authtypes.Device, error) - GetDevicesByLocalpart(ctx context.Context, localpart string) ([]authtypes.Device, error) - CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *authtypes.Device, returnErr error) - UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error - RemoveDevice(ctx context.Context, deviceID, localpart string) error - RemoveDevices(ctx context.Context, localpart string, devices []string) error - RemoveAllDevices(ctx context.Context, localpart string) error -} - func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go new file mode 100644 index 00000000..32285288 --- /dev/null +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devices + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 6df0b28b..ba24e527 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -444,7 +444,6 @@ func Register( deviceDB devices.Database, cfg *config.Dendrite, ) util.JSONResponse { - var r registerRequest resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go new file mode 100644 index 00000000..347a5283 --- /dev/null +++ b/cmd/dendritejs/main.go @@ -0,0 +1,160 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "crypto/ed25519" + "fmt" + "net/http" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/mediaapi" + "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/syncapi" + "github.com/matrix-org/dendrite/typingserver" + "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/sirupsen/logrus" + + _ "github.com/matrix-org/go-sqlite3-js" +) + +func init() { + fmt.Println("dendrite.js starting...") +} + +func generateKey() ed25519.PrivateKey { + _, priv, err := ed25519.GenerateKey(nil) + if err != nil { + logrus.Fatalf("Failed to generate ed25519 key: %s", err) + } + return priv +} + +func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.FederationClient { + fmt.Println("Running in js-libp2p federation mode") + fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!") + tr := go_http_js_libp2p.NewP2pTransport(node) + + fed := gomatrixserverlib.NewFederationClient( + cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, + ) + fed.Client = *gomatrixserverlib.NewClientWithTransport(tr) + + return fed +} + +func createP2PNode() (serverName string, node *go_http_js_libp2p.P2pLocalNode) { + hosted := "/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star" + _ = "/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/" + node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", []string{hosted}) + serverName = node.Id + fmt.Println("p2p assigned ServerName: ", serverName) + return +} + +func main() { + cfg := &config.Dendrite{} + cfg.SetDefaults() + cfg.Kafka.UseNaffka = true + cfg.Database.Account = "file:dendritejs_account.db" + cfg.Database.AppService = "file:dendritejs_appservice.db" + cfg.Database.Device = "file:dendritejs_device.db" + cfg.Database.FederationSender = "file:dendritejs_fedsender.db" + cfg.Database.MediaAPI = "file:dendritejs_mediaapi.db" + cfg.Database.Naffka = "file:dendritejs_naffka.db" + cfg.Database.PublicRoomsAPI = "file:dendritejs_publicrooms.db" + cfg.Database.RoomServer = "file:dendritejs_roomserver.db" + cfg.Database.ServerKey = "file:dendritejs_serverkey.db" + cfg.Database.SyncAPI = "file:dendritejs_syncapi.db" + cfg.Matrix.TrustedIDServers = []string{ + "matrix.org", "vector.im", + } + cfg.Matrix.KeyID = "ed25519:1337" + cfg.Matrix.PrivateKey = generateKey() + + serverName, node := createP2PNode() + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(serverName) + + if err := cfg.Derive(); err != nil { + logrus.Fatalf("Failed to derive values from config: %s", err) + } + base := basecomponent.NewBaseDendrite(cfg, "Monolith") + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + keyDB := base.CreateKeyDB() + federation := createFederationClient(cfg, node) + keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + + alias, input, query := roomserver.SetupRoomServerComponent(base) + typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) + asQuery := appservice.SetupAppServiceAPIComponent( + base, accountDB, deviceDB, federation, alias, query, transactions.New(), + ) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + + clientapi.SetupClientAPIComponent( + base, deviceDB, accountDB, + federation, &keyRing, alias, input, query, + typingInputAPI, asQuery, transactions.New(), fedSenderAPI, + ) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) + mediaapi.SetupMediaAPIComponent(base, deviceDB) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + + httpHandler := common.WrapHandlerInCORS(base.APIMux) + + http.Handle("/", httpHandler) + + // Expose the matrix APIs via libp2p-js - for federation traffic + if node != nil { + go func() { + logrus.Info("Listening on libp2p-js host ID ", node.Id) + + listener := go_http_js_libp2p.NewP2pListener(node) + defer listener.Close() + s := &http.Server{} + s.Serve(listener) + }() + } + + // Expose the matrix APIs via fetch - for local traffic + go func() { + logrus.Info("Listening for service-worker fetch traffic") + + listener := go_http_js_libp2p.NewFetchListener() + s := &http.Server{} + go s.Serve(listener) + }() + + // We want to block forever to let the fetch and libp2p handler serve the APIs + select {} +} diff --git a/cmd/dendritejs/main_noop.go b/cmd/dendritejs/main_noop.go new file mode 100644 index 00000000..dcea032f --- /dev/null +++ b/cmd/dendritejs/main_noop.go @@ -0,0 +1,23 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package main + +import "fmt" + +func main() { + fmt.Println("dendritejs: no-op when not compiling for WebAssembly") +} diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 4274de2b..d1d953f7 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -216,7 +216,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sql.Open("sqlite3", string(cfg.Database.Naffka)) + db, err = sql.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/config/config.go b/common/config/config.go index 0332d035..bd83cbf8 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -224,6 +224,8 @@ type Dendrite struct { // The config for tracing the dendrite servers. Tracing struct { + // Set to true to enable tracer hooks. If false, no tracing is set up. + Enabled bool `yaml:"enabled"` // The config for the jaeger opentracing reporter. Jaeger jaegerconfig.Configuration `yaml:"jaeger"` } `yaml:"tracing"` @@ -365,7 +367,7 @@ func loadConfig( return nil, err } - config.setDefaults() + config.SetDefaults() if err = config.check(monolithic); err != nil { return nil, err @@ -398,7 +400,7 @@ func loadConfig( config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) // Generate data from config options - err = config.derive() + err = config.Derive() if err != nil { return nil, err } @@ -406,9 +408,9 @@ func loadConfig( return &config, nil } -// derive generates data that is derived from various values provided in +// Derive generates data that is derived from various values provided in // the config file. -func (config *Dendrite) derive() error { +func (config *Dendrite) Derive() error { // Determine registrations flows based off config values config.Derived.Registration.Params = make(map[string]interface{}) @@ -433,8 +435,8 @@ func (config *Dendrite) derive() error { return nil } -// setDefaults sets default config values if they are not explicitly set. -func (config *Dendrite) setDefaults() { +// SetDefaults sets default config values if they are not explicitly set. +func (config *Dendrite) SetDefaults() { if config.Matrix.KeyValidityPeriod == 0 { config.Matrix.KeyValidityPeriod = 24 * time.Hour } @@ -703,6 +705,9 @@ func (config *Dendrite) FederationSenderURL() string { // SetupTracing configures the opentracing using the supplied configuration. func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { + if !config.Tracing.Enabled { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } return config.Tracing.Jaeger.InitGlobalTracer( serviceName, jaegerconfig.Logger(logrusLogger{logrus.StandardLogger()}), diff --git a/common/keydb/interface.go b/common/keydb/interface.go new file mode 100644 index 00000000..c9a20fdd --- /dev/null +++ b/common/keydb/interface.go @@ -0,0 +1,13 @@ +package keydb + +import ( + "context" + + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + FetcherName() string + FetchKeys(ctx context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) + StoreKeys(ctx context.Context, keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error +} diff --git a/common/keydb/keydb.go b/common/keydb/keydb.go index cf15c9f0..fe6d87fc 100644 --- a/common/keydb/keydb.go +++ b/common/keydb/keydb.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package keydb import ( - "context" "net/url" "golang.org/x/crypto/ed25519" @@ -25,12 +26,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - FetcherName() string - FetchKeys(ctx context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) - StoreKeys(ctx context.Context, keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error -} - // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, diff --git a/common/keydb/keydb_wasm.go b/common/keydb/keydb_wasm.go new file mode 100644 index 00000000..807ed40b --- /dev/null +++ b/common/keydb/keydb_wasm.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keydb + +import ( + "fmt" + "net/url" + + "golang.org/x/crypto/ed25519" + + "github.com/matrix-org/dendrite/common/keydb/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +// NewDatabase opens a database connection. +func NewDatabase( + dataSourceName string, + serverName gomatrixserverlib.ServerName, + serverKey ed25519.PublicKey, + serverKeyID gomatrixserverlib.KeyID, +) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 88eb9d9f..3c860d0c 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -22,6 +22,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -43,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sql.Open("sqlite3", dataSourceName) + db, err := sql.Open(common.SQLiteDriverName(), dataSourceName) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go index 6c33f30a..3caf4ea4 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/common/keydb/sqlite3/server_key_table.go @@ -18,9 +18,12 @@ package sqlite3 import ( "context" "database/sql" + "strings" - "github.com/lib/pq" + lru "github.com/hashicorp/golang-lru" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) const serverKeysSchema = ` @@ -60,11 +63,19 @@ const upsertServerKeysSQL = "" + " DO UPDATE SET valid_until_ts = $4, expired_ts = $5, server_key = $6" type serverKeyStatements struct { + db *sql.DB bulkSelectServerKeysStmt *sql.Stmt upsertServerKeysStmt *sql.Stmt + + cache *lru.Cache // nameAndKeyID => gomatrixserverlib.PublicKeyLookupResult } func (s *serverKeyStatements) prepare(db *sql.DB) (err error) { + s.db = db + s.cache, err = lru.New(64) + if err != nil { + return + } _, err = db.Exec(serverKeysSchema) if err != nil { return @@ -86,8 +97,30 @@ func (s *serverKeyStatements) bulkSelectServerKeys( for request := range requests { nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) } - stmt := s.bulkSelectServerKeysStmt - rows, err := stmt.QueryContext(ctx, pq.StringArray(nameAndKeyIDs)) + + // If we can satisfy all of the requests from the cache, do so. TODO: Allow partial matches with merges. + cacheResults := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + for request := range requests { + r, ok := s.cache.Get(nameAndKeyID(request)) + if !ok { + break + } + cacheResult := r.(gomatrixserverlib.PublicKeyLookupResult) + cacheResults[request] = cacheResult + } + if len(cacheResults) == len(requests) { + util.GetLogger(ctx).Infof("KeyDB cache hit for %d keys", len(cacheResults)) + return cacheResults, nil + } + + query := strings.Replace(bulkSelectServerKeysSQL, "($1)", common.QueryVariadic(len(nameAndKeyIDs)), 1) + + iKeyIDs := make([]interface{}, len(nameAndKeyIDs)) + for i, v := range nameAndKeyIDs { + iKeyIDs[i] = v + } + + rows, err := s.db.QueryContext(ctx, query, iKeyIDs...) if err != nil { return nil, err } @@ -125,6 +158,7 @@ func (s *serverKeyStatements) upsertServerKeys( request gomatrixserverlib.PublicKeyLookupRequest, key gomatrixserverlib.PublicKeyLookupResult, ) error { + s.cache.Add(nameAndKeyID(request), key) _, err := s.upsertServerKeysStmt.ExecContext( ctx, string(request.ServerName), diff --git a/common/postgres.go b/common/postgres.go new file mode 100644 index 00000000..f8daf578 --- /dev/null +++ b/common/postgres.go @@ -0,0 +1,25 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package common + +import "github.com/lib/pq" + +// IsUniqueConstraintViolationErr returns true if the error is a postgresql unique_violation error +func IsUniqueConstraintViolationErr(err error) bool { + pqErr, ok := err.(*pq.Error) + return ok && pqErr.Code == "23505" +} diff --git a/common/postgres_wasm.go b/common/postgres_wasm.go new file mode 100644 index 00000000..dcc07b31 --- /dev/null +++ b/common/postgres_wasm.go @@ -0,0 +1,22 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package common + +// IsUniqueConstraintViolationErr no-ops for this architecture +func IsUniqueConstraintViolationErr(err error) bool { + return false +} diff --git a/common/sql.go b/common/sql.go index 97593020..f50a5896 100644 --- a/common/sql.go +++ b/common/sql.go @@ -17,8 +17,7 @@ package common import ( "database/sql" "fmt" - - "github.com/lib/pq" + "runtime" ) // A Transaction is something that can be committed or rolledback. @@ -77,12 +76,6 @@ func TxStmt(transaction *sql.Tx, statement *sql.Stmt) *sql.Stmt { return statement } -// IsUniqueConstraintViolationErr returns true if the error is a postgresql unique_violation error -func IsUniqueConstraintViolationErr(err error) bool { - pqErr, ok := err.(*pq.Error) - return ok && pqErr.Code == "23505" -} - // Hack of the century func QueryVariadic(count int) string { return QueryVariadicOffset(count, 0) @@ -99,3 +92,10 @@ func QueryVariadicOffset(count, offset int) string { str += ")" return str } + +func SQLiteDriverName() string { + if runtime.GOOS == "js" { + return "sqlite3_js" + } + return "sqlite3" +} diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go new file mode 100644 index 00000000..ae295647 --- /dev/null +++ b/federationsender/storage/interface.go @@ -0,0 +1,28 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/federationsender/types" +) + +type Database interface { + common.PartitionStorer + UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) + GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) +} diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index f9cfaa99..6ab9ae00 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index e83c1e9d..2f018dff 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -12,24 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/federationsender/types" ) -type Database interface { - common.PartitionStorer - UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) - GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) -} - // NewDatabase opens a new database func NewDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go new file mode 100644 index 00000000..f2c8ae1b --- /dev/null +++ b/federationsender/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" +) + +// NewDatabase opens a new database +func NewDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "file": + return sqlite3.NewDatabase(dataSourceName) + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/go.mod b/go.mod index 2d442cd0..7d59aeb1 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,30 @@ module github.com/matrix-org/dendrite require ( - github.com/DataDog/zstd v1.4.4 // indirect - github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect - github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/eapache/go-resiliency v1.2.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect - github.com/eapache/queue v1.1.0 // indirect - github.com/frankban/quicktest v1.7.2 // indirect - github.com/golang/snappy v0.0.1 // indirect github.com/gorilla/mux v1.7.3 - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 + github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 - github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 + github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible - github.com/miekg/dns v1.1.12 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 - github.com/opentracing/opentracing-go v1.0.2 - github.com/pierrec/lz4 v2.4.1+incompatible // indirect + github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 - github.com/prometheus/client_golang v1.2.1 - github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect + github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.4.0 // indirect - github.com/uber-go/atomic v1.3.0 // indirect - github.com/uber/jaeger-client-go v2.15.0+incompatible - github.com/uber/jaeger-lib v1.5.0 - go.uber.org/atomic v1.3.0 // indirect + github.com/tidwall/gjson v1.6.0 + github.com/tidwall/pretty v1.0.1 // indirect + github.com/uber/jaeger-client-go v2.22.1+incompatible + github.com/uber/jaeger-lib v2.2.0+incompatible golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 - golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect - golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect gopkg.in/Shopify/sarama.v1 v1.20.1 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 - gopkg.in/yaml.v2 v2.2.2 + gopkg.in/yaml.v2 v2.2.5 ) go 1.13 diff --git a/go.sum b/go.sum index 7c8732f6..19dc9662 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= @@ -40,17 +41,20 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= @@ -63,18 +67,37 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94 h1:gUqPXKWbuwmPnx3PJBxtQw5Ff8V229wZ7Ee0GaH/bg4= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 h1:oFjG7X1jS473zPDix1/FBZ2qd0anM1Ko4AlJCB6MUZs= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8 h1:a9IV2iKMznKJ16MmcG/NU7qMcZ4jIKmPXC6RkOZZq+Q= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e h1:DA1lP2mB2ddd2PhMOaNPwRJFi/3aL2Lj7bxnAhFxBFQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= +github.com/matrix-org/pq v1.3.2 h1:7Kh2Qz4xonRH8OcFpCIj2najpBxF2+j1ff0hC082L68= +github.com/matrix-org/pq v1.3.2/go.mod h1:l6tPTzDjcj8fhD5OD0+7dejtZrwyjJadAAYM+CntUVQ= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= +github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= +github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -94,6 +117,8 @@ github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/R github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -103,19 +128,23 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= +github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -126,22 +155,30 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= +github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= +github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= +github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= +github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -156,24 +193,26 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -181,6 +220,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= @@ -190,3 +230,6 @@ gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/mediaapi/storage/interface.go b/mediaapi/storage/interface.go new file mode 100644 index 00000000..672e8ef5 --- /dev/null +++ b/mediaapi/storage/interface.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error + GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) + StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error + GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) + GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) +} diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index 6477f830..cfa5bb98 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" // Import the postgres database driver. + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -35,7 +36,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 1d35a95e..c533477c 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -12,26 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" - "github.com/matrix-org/dendrite/mediaapi/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error - GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) - StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error - GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) - GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) -} - // Open opens a postgres database. func Open(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go new file mode 100644 index 00000000..92f0ad13 --- /dev/null +++ b/mediaapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" +) + +// Open opens a postgres database. +func Open(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.Open(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/p2p.md b/p2p.md new file mode 100644 index 00000000..0c3dc83b --- /dev/null +++ b/p2p.md @@ -0,0 +1,81 @@ +## Peer-to-peer Matrix + +These are the instructions for setting up P2P Dendrite, current as of March 2020. There's both Go stuff and JS stuff to do to set this up. + + +### Dendrite + +- The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`. +- Build it and copy assets to riot-web. +``` +$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs +$ cp main.wasm ../riot-web/src/vector/dendrite.wasm +``` + +### Rendezvous + +This is how peers discover each other and communicate. + +By default, Dendrite uses the IPFS-hosted websocket star **Development** relay server at `/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star`. +This is currently hard-coded in `./cmd/dendritejs/main.go` - you can also use a local one if you run your own relay: + +``` +$ npm install --global libp2p-websocket-star-rendezvous +$ rendezvous --port=9090 --host=127.0.0.1 +``` + +Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`. We'll probably run our own relay server at some point. + +### Riot-web + +You need to check out these repos: + +`` +$ git clone git@github.com:matrix-org/go-http-js-libp2p.git +$ git clone git@github.com:matrix-org/go-sqlite3-js.git +``` + +Make sure to `yarn install` in both of these repos. Then: + +- `$ cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" ./src/vector/` +- Comment out the lines in `wasm_exec.js` which contains: +``` +if (!global.fs && global.require) { + global.fs = require("fs"); +} +``` +- Add the diff at https://github.com/vector-im/riot-web/compare/matthew/p2p?expand=1 - ignore the `package.json` stuff. +- Add the following symlinks: they HAVE to be symlinks as the diff in `webpack.config.js` references specific paths. +``` +$ cd node_modules +$ ln -s ../../go-sqlite-js # NB: NOT go-sqlite3-js +$ ln -s ../../go-http-js-libp2p +``` + +NB: If you don't run the server with `yarn start` you need to make sure your server is sending the header `Service-Worker-Allowed: /`. + +TODO: Make a Docker image with all of this in it and a volume mount for `dendrite.wasm`. + +## Running + +You need a Chrome and a Firefox running to test locally as service workers don't work in incognito tabs. +- For Chrome, use `chrome://serviceworker-internals/` to unregister/see logs. +- For Firefox, use `about:debugging#/runtime/this-firefox` to unregister. Use the console window to see logs. + +Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and wait a bit. Then refresh the page (this is required +because the fetch interceptor races with setting up dendrite. If you don't refresh, you won't be able to contact your HS). After +the refresh, click Register and use `http://localhost:8080` as your HS URL. + +TODO: Fix the race so we don't need multiple refreshes. + +You can join rooms by room alias e.g `/join #foo:bar`. + +### Known issues + +- When registering you may be unable to find the server, it'll seem flakey. This happens because the SW, particularly in Firefox, + gets killed after 30s of inactivity. When you are not registered, you aren't doing `/sync` calls to keep the SW alive, so if you + don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page *twice*. + +- The libp2p layer has rate limits, so frequent Federation traffic may cause the connection to drop and messages to not be transferred. + I guess in other words, don't send too much traffic? + diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go new file mode 100644 index 00000000..3f7e6589 --- /dev/null +++ b/publicroomsapi/storage/interface.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + GetRoomVisibility(ctx context.Context, roomID string) (bool, error) + SetRoomVisibility(ctx context.Context, visible bool, roomID string) error + CountPublicRooms(ctx context.Context) (int64, error) + GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) + UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error + UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error +} diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go index 3fe3d3ca..beba0d69 100644 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "errors" "fmt" - "github.com/lib/pq" "github.com/matrix-org/dendrite/publicroomsapi/types" ) @@ -196,17 +196,21 @@ func (s *publicRoomsStatements) selectPublicRooms( rooms := []types.PublicRoom{} for rows.Next() { var r types.PublicRoom - var aliases pq.StringArray + var aliasesJSON string err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias, + &r.RoomID, &r.NumJoinedMembers, &aliasesJSON, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { return rooms, err } - r.Aliases = aliases + if len(aliasesJSON) > 0 { + if err := json.Unmarshal([]byte(aliasesJSON), &r.Aliases); err != nil { + return rooms, err + } + } rooms = append(rooms, r) } @@ -254,7 +258,8 @@ func (s *publicRoomsStatements) updateRoomAttribute( var value interface{} switch v := attrValue.(type) { case []string: - value = pq.StringArray(v) + b, _ := json.Marshal(v) + value = string(b) case bool, string: value = attrValue default: diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index dcb8920f..f8ba71a8 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -41,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index 29a6619f..41dbd781 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -12,29 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/dendrite/publicroomsapi/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - GetRoomVisibility(ctx context.Context, roomID string) (bool, error) - SetRoomVisibility(ctx context.Context, visible bool, roomID string) error - CountPublicRooms(ctx context.Context) (int64, error) - GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) - UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error - UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error -} - // NewPublicRoomsServerDatabase opens a database connection. func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go new file mode 100644 index 00000000..d00c339d --- /dev/null +++ b/publicroomsapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go new file mode 100644 index 00000000..f03d6fea --- /dev/null +++ b/roomserver/storage/interface.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/api" + statedb "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + statedb.RoomStateDatabase + StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) + StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) + EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) + EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) + SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error + EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) + GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) + GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) + RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) + LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) + GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) + SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error + GetRoomIDForAlias(ctx context.Context, alias string) (string, error) + GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) + GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) + RemoveRoomAlias(ctx context.Context, alias string) error + MembershipUpdater(ctx context.Context, roomID, targetUserID string) (types.MembershipUpdater, error) + GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) + GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) + EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) + GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) +} diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 4ed1395d..56e37910 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "fmt" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -38,7 +38,7 @@ const eventsSchema = ` depth INTEGER NOT NULL, event_id TEXT NOT NULL UNIQUE, reference_sha256 BLOB NOT NULL, - auth_event_nids TEXT NOT NULL DEFAULT '{}' + auth_event_nids TEXT NOT NULL DEFAULT '[]' ); ` @@ -463,17 +463,14 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { var result int64 selectStmt := common.TxStmt(txn, s.selectMaxEventDepthStmt) - err := selectStmt.QueryRowContext(ctx, sqliteIn(eventNIDsAsArray(eventNIDs))).Scan(&result) + err := selectStmt.QueryRowContext(ctx, eventNIDsAsArray(eventNIDs)).Scan(&result) if err != nil { return 0, err } return result, nil } -func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { - nids := make([]int64, len(eventNIDs)) - for i := range eventNIDs { - nids[i] = int64(eventNIDs[i]) - } - return nids +func eventNIDsAsArray(eventNIDs []types.EventNID) string { + b, _ := json.Marshal(eventNIDs) + return string(b) } diff --git a/roomserver/storage/sqlite3/list.go b/roomserver/storage/sqlite3/list.go deleted file mode 100644 index 4fe4e334..00000000 --- a/roomserver/storage/sqlite3/list.go +++ /dev/null @@ -1,18 +0,0 @@ -package sqlite3 - -import ( - "strconv" - "strings" - - "github.com/lib/pq" -) - -type SqliteList string - -func sqliteIn(a pq.Int64Array) string { - var b []string - for _, n := range a { - b = append(b, strconv.FormatInt(n, 10)) - } - return strings.Join(b, ",") -} diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index bf237728..99c03476 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -18,8 +18,8 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -28,7 +28,7 @@ const roomsSchema = ` CREATE TABLE IF NOT EXISTS roomserver_rooms ( room_nid INTEGER PRIMARY KEY AUTOINCREMENT, room_id TEXT NOT NULL UNIQUE, - latest_event_nids TEXT NOT NULL DEFAULT '{}', + latest_event_nids TEXT NOT NULL DEFAULT '[]', last_event_sent_nid INTEGER NOT NULL DEFAULT 0, state_snapshot_nid INTEGER NOT NULL DEFAULT 0, room_version INTEGER NOT NULL DEFAULT 1 @@ -104,16 +104,16 @@ func (s *roomStatements) selectRoomNID( func (s *roomStatements) selectLatestEventNIDs( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.StateSnapshotNID, error) { - var nids pq.Int64Array + var eventNIDs []types.EventNID + var nidsJSON string var stateSnapshotNID int64 stmt := common.TxStmt(txn, s.selectLatestEventNIDsStmt) - err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &stateSnapshotNID) + err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &stateSnapshotNID) if err != nil { return nil, 0, err } - eventNIDs := make([]types.EventNID, len(nids)) - for i := range nids { - eventNIDs[i] = types.EventNID(nids[i]) + if err := json.Unmarshal([]byte(nidsJSON), &eventNIDs); err != nil { + return nil, 0, err } return eventNIDs, types.StateSnapshotNID(stateSnapshotNID), nil } @@ -121,17 +121,17 @@ func (s *roomStatements) selectLatestEventNIDs( func (s *roomStatements) selectLatestEventsNIDsForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.EventNID, types.StateSnapshotNID, error) { - var nids pq.Int64Array + var eventNIDs []types.EventNID + var nidsJSON string var lastEventSentNID int64 var stateSnapshotNID int64 stmt := common.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) - err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &lastEventSentNID, &stateSnapshotNID) + err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err } - eventNIDs := make([]types.EventNID, len(nids)) - for i := range nids { - eventNIDs[i] = types.EventNID(nids[i]) + if err := json.Unmarshal([]byte(nidsJSON), &eventNIDs); err != nil { + return nil, 0, 0, err } return eventNIDs, types.EventNID(lastEventSentNID), types.StateSnapshotNID(stateSnapshotNID), nil } diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index d75abcee..c94ca238 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -22,7 +22,6 @@ import ( "sort" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -268,9 +267,9 @@ func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool { // List the unique eventTypeNIDs and eventStateKeyNIDs. // Assumes that the list is sorted. -func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs pq.Int64Array, eventStateKeyNIDs pq.Int64Array) { - eventTypeNIDs = make(pq.Int64Array, len(s)) - eventStateKeyNIDs = make(pq.Int64Array, len(s)) +func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) { + eventTypeNIDs = make([]int64, len(s)) + eventStateKeyNIDs = make([]int64, len(s)) for i := range s { eventTypeNIDs[i] = int64(s[i].EventTypeNID) eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID) diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index df97aa41..0e1786ea 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "fmt" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -30,7 +30,7 @@ const stateSnapshotSchema = ` CREATE TABLE IF NOT EXISTS roomserver_state_snapshots ( state_snapshot_nid INTEGER PRIMARY KEY AUTOINCREMENT, room_nid INTEGER NOT NULL, - state_block_nids TEXT NOT NULL DEFAULT '{}' + state_block_nids TEXT NOT NULL DEFAULT '[]' ); ` @@ -67,12 +67,12 @@ func (s *stateSnapshotStatements) prepare(db *sql.DB) (err error) { func (s *stateSnapshotStatements) insertState( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, ) (stateNID types.StateSnapshotNID, err error) { - nids := make([]int64, len(stateBlockNIDs)) - for i := range stateBlockNIDs { - nids[i] = int64(stateBlockNIDs[i]) + stateBlockNIDsJSON, err := json.Marshal(stateBlockNIDs) + if err != nil { + return } insertStmt := txn.Stmt(s.insertStateStmt) - if res, err2 := insertStmt.ExecContext(ctx, int64(roomNID), pq.Int64Array(nids)); err2 == nil { + if res, err2 := insertStmt.ExecContext(ctx, int64(roomNID), string(stateBlockNIDsJSON)); err2 == nil { lastRowID, err3 := res.LastInsertId() if err3 != nil { err = err3 @@ -104,13 +104,12 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( i := 0 for ; rows.Next(); i++ { result := &results[i] - var stateBlockNIDs pq.Int64Array - if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil { + var stateBlockNIDsJSON string + if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { return nil, err } - result.StateBlockNIDs = make([]types.StateBlockNID, len(stateBlockNIDs)) - for k := range stateBlockNIDs { - result.StateBlockNIDs[k] = types.StateBlockNID(stateBlockNIDs[k]) + if err := json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { + return nil, err } } if i != len(stateNIDs) { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index aebb308c..41d4c2a9 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -49,7 +49,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open("sqlite3", cs); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 551d97cd..7b9109aa 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -12,45 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/roomserver/api" - statedb "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - statedb.RoomStateDatabase - StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) - StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) - EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) - EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) - SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error - EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) - GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) - GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) - RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) - LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) - GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) - SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error - GetRoomIDForAlias(ctx context.Context, alias string) (string, error) - GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) - GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) - RemoveRoomAlias(ctx context.Context, alias string) error - MembershipUpdater(ctx context.Context, roomID, targetUserID string) (types.MembershipUpdater, error) - GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) - GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) - EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) - GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) -} - // NewPublicRoomsServerDatabase opens a database connection. func Open(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go new file mode 100644 index 00000000..d7fc352e --- /dev/null +++ b/roomserver/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func Open(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.Open(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go new file mode 100644 index 00000000..0abeac34 --- /dev/null +++ b/syncapi/storage/interface.go @@ -0,0 +1,53 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) + Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) + WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) + SyncPosition(ctx context.Context) (types.PaginationToken, error) + IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) + GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) + AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) + RetireInviteEvent(ctx context.Context, inviteEventID string) error + SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) + AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition + RemoveTypingUser(userID, roomID string) types.StreamPosition + GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) + EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) + BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) + MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event + SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) +} diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index eb969c95..ed76177b 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -21,7 +21,6 @@ import ( "encoding/json" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -175,10 +174,10 @@ func (s *currentRoomStateStatements) selectCurrentState( ) ([]gomatrixserverlib.Event, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, - pq.StringArray(stateFilterPart.Senders), - pq.StringArray(stateFilterPart.NotSenders), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + nil, // FIXME: pq.StringArray(stateFilterPart.Senders), + nil, // FIXME: pq.StringArray(stateFilterPart.NotSenders), + nil, // FIXME: pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + nil, // FIXME: pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), stateFilterPart.ContainsURL, stateFilterPart.Limit, ) diff --git a/syncapi/storage/sqlite3/filtering.go b/syncapi/storage/sqlite3/filtering.go deleted file mode 100644 index c4a2f4bf..00000000 --- a/syncapi/storage/sqlite3/filtering.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Thibaut CHARLES -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "strings" -) - -// filterConvertWildcardToSQL converts wildcards as defined in -// https://matrix.org/docs/spec/client_server/r0.3.0.html#post-matrix-client-r0-user-userid-filter -// to SQL wildcards that can be used with LIKE() -func filterConvertTypeWildcardToSQL(values []string) []string { - if values == nil { - // Return nil instead of []string{} so IS NULL can work correctly when - // the return value is passed into SQL queries - return nil - } - - ret := make([]string, len(values)) - for i := range values { - ret[i] = strings.Replace(values[i], "*", "%", -1) - } - return ret -} diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 4535688d..be893743 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" @@ -40,11 +39,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( type TEXT NOT NULL, sender TEXT NOT NULL, contains_url BOOL NOT NULL, - add_state_ids TEXT[], - remove_state_ids TEXT[], + add_state_ids TEXT, -- JSON encoded string array + remove_state_ids TEXT, -- JSON encoded string array session_id BIGINT, transaction_id TEXT, - exclude_from_sync BOOL DEFAULT FALSE + exclude_from_sync BOOL NOT NULL DEFAULT FALSE ); ` @@ -52,7 +51,7 @@ const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + "id, room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + - "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $11" + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $13" const selectEventsSQL = "" + "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" @@ -176,20 +175,26 @@ func (s *outputRoomEventsStatements) selectStateInRange( streamPos types.StreamPosition eventBytes []byte excludeFromSync bool - addIDs pq.StringArray - delIDs pq.StringArray + addIDsJSON string + delIDsJSON string ) - if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { + if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil { return nil, nil, err } + + addIDs, delIDs, err := unmarshalStateIDs(addIDsJSON, delIDsJSON) + if err != nil { + return nil, nil, err + } + // Sanity check for deleted state and whine if we see it. We don't need to do anything // since it'll just mark the event as not being needed. if len(addIDs) < len(delIDs) { log.WithFields(log.Fields{ "since": oldPos, "current": newPos, - "adds": addIDs, - "dels": delIDs, + "adds": addIDsJSON, + "dels": delIDsJSON, }).Warn("StateBetween: ignoring deleted state") } @@ -262,6 +267,15 @@ func (s *outputRoomEventsStatements) insertEvent( return } + addStateJSON, err := json.Marshal(addState) + if err != nil { + return + } + removeStateJSON, err := json.Marshal(removeState) + if err != nil { + return + } + insertStmt := common.TxStmt(txn, s.insertEventStmt) _, err = insertStmt.ExecContext( ctx, @@ -272,11 +286,12 @@ func (s *outputRoomEventsStatements) insertEvent( event.Type(), event.Sender(), containsURL, - pq.StringArray(addState), - pq.StringArray(removeState), + string(addStateJSON), + string(removeStateJSON), sessionID, txnID, excludeFromSync, + excludeFromSync, ) return } @@ -397,3 +412,17 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } return result, nil } + +func unmarshalStateIDs(addIDsJSON, delIDsJSON string) (addIDs []string, delIDs []string, err error) { + if len(addIDsJSON) > 0 { + if err = json.Unmarshal([]byte(addIDsJSON), &addIDs); err != nil { + return + } + } + if len(delIDsJSON) > 0 { + if err = json.Unmarshal([]byte(delIDsJSON), &delIDs); err != nil { + return + } + } + return +} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index da580e3a..0e84c8c8 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -29,8 +29,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/roomserver/api" - // Import the postgres database driver. - _ "github.com/lib/pq" + // Import the sqlite3 package _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" @@ -79,7 +78,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open("sqlite3", cs); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } if err = d.prepare(); err != nil { diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index c87024b2..c56db063 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -12,49 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" - "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/typingserver/cache" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) - Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) - WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) - GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) - SyncPosition(ctx context.Context) (types.PaginationToken, error) - IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) - CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) - GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) - UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) - AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) - RetireInviteEvent(ctx context.Context, inviteEventID string) error - SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) - AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition - RemoveTypingUser(userID, roomID string) types.StreamPosition - GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) - EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) - EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) - BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) - MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) - StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event - SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) -} - // NewPublicRoomsServerDatabase opens a database connection. func NewSyncServerDatasource(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go new file mode 100644 index 00000000..43806a01 --- /dev/null +++ b/syncapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func NewSyncServerDatasource(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewSyncServerDatasource(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +}