2020-02-13 17:27:33 +00:00
|
|
|
// Copyright 2017-2018 New Vector Ltd
|
|
|
|
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package sqlite3
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"database/sql"
|
Add peer-to-peer support into Dendrite via libp2p and fetch (#880)
* Use a fork of pq which supports userCurrent on wasm
* Use sqlite3_js driver when running in JS
* Add cmd/dendritejs to pull in sqlite3_js driver for wasm only
* Update to latest go-sqlite-js version
* Replace prometheus with a stub. sigh
* Hard-code a config and don't use opentracing
* Latest go-sqlite3-js version
* Generate a key for now
* Listen for fetch traffic rather than HTTP
* Latest hacks for js
* libp2p support
* More libp2p
* Fork gjson to allow us to enforce auth checks as before
Previously, all events would come down redacted because the hash
checks would fail. They would fail because sjson.DeleteBytes didn't
remove keys not used for hashing. This didn't work because of a build
tag which included a file which no-oped the index returned.
See https://github.com/tidwall/gjson/issues/157
When it's resolved, let's go back to mainline.
* Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157
* Use latest gomatrixserverlib for sig checks
* Fix a bug which could cause exclude_from_sync to not be set
Caused when sending events over federation.
* Use query variadic to make lookups actually work!
* Latest gomatrixserverlib
* Add notes on getting p2p up and running
Partly so I don't forget myself!
* refactor: Move p2p specific stuff to cmd/dendritejs
This is important or else the normal build of dendrite will fail
because the p2p libraries depend on syscall/js which doesn't work
on normal builds.
Also, clean up main.go to read a bit better.
* Update ho-http-js-libp2p to return errors from RoundTrip
* Add an LRU cache around the key DB
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.
Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
* breaking: Add Tracing.Enabled to toggle whether we do opentracing
Defaults to false, which is why this is a breaking change. We need
this flag because WASM builds cannot do opentracing.
* Start adding conditional builds for wasm to handle lib/pq
The general idea here is to have the wasm build have a `NewXXXDatabase`
that doesn't import any postgres package and hence we never import
`lib/pq`, which doesn't work under WASM (undefined `userCurrent`).
* Remove lib/pq for wasm for syncapi
* Add conditional building to remaining storage APIs
* Update build script to set env vars correctly for dendritejs
* sqlite bug fixes
* Docs
* Add a no-op main for dendritejs when not building under wasm
* Use the real prometheus, even for WASM
Instead, the dendrite-sw.js must mock out `process.pid` and
`fs.stat` - which must invoke the callback with an error (e.g `EINVAL`)
in order for it to work:
```
global.process = {
pid: 1,
};
global.fs.stat = function(path, cb) {
cb({
code: "EINVAL",
});
}
```
* Linting
2020-03-06 10:23:55 +00:00
|
|
|
"encoding/json"
|
2020-07-21 15:48:21 +01:00
|
|
|
"fmt"
|
2020-09-03 17:20:54 +01:00
|
|
|
"strings"
|
2020-02-13 17:27:33 +00:00
|
|
|
|
2020-09-03 17:20:54 +01:00
|
|
|
"github.com/matrix-org/dendrite/internal"
|
2020-06-12 14:55:57 +01:00
|
|
|
"github.com/matrix-org/dendrite/internal/sqlutil"
|
2020-05-26 18:23:39 +01:00
|
|
|
"github.com/matrix-org/dendrite/roomserver/storage/tables"
|
2020-02-13 17:27:33 +00:00
|
|
|
"github.com/matrix-org/dendrite/roomserver/types"
|
2020-03-16 16:05:29 +00:00
|
|
|
"github.com/matrix-org/gomatrixserverlib"
|
2020-02-13 17:27:33 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const roomsSchema = `
|
|
|
|
CREATE TABLE IF NOT EXISTS roomserver_rooms (
|
|
|
|
room_nid INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
|
room_id TEXT NOT NULL UNIQUE,
|
Add peer-to-peer support into Dendrite via libp2p and fetch (#880)
* Use a fork of pq which supports userCurrent on wasm
* Use sqlite3_js driver when running in JS
* Add cmd/dendritejs to pull in sqlite3_js driver for wasm only
* Update to latest go-sqlite-js version
* Replace prometheus with a stub. sigh
* Hard-code a config and don't use opentracing
* Latest go-sqlite3-js version
* Generate a key for now
* Listen for fetch traffic rather than HTTP
* Latest hacks for js
* libp2p support
* More libp2p
* Fork gjson to allow us to enforce auth checks as before
Previously, all events would come down redacted because the hash
checks would fail. They would fail because sjson.DeleteBytes didn't
remove keys not used for hashing. This didn't work because of a build
tag which included a file which no-oped the index returned.
See https://github.com/tidwall/gjson/issues/157
When it's resolved, let's go back to mainline.
* Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157
* Use latest gomatrixserverlib for sig checks
* Fix a bug which could cause exclude_from_sync to not be set
Caused when sending events over federation.
* Use query variadic to make lookups actually work!
* Latest gomatrixserverlib
* Add notes on getting p2p up and running
Partly so I don't forget myself!
* refactor: Move p2p specific stuff to cmd/dendritejs
This is important or else the normal build of dendrite will fail
because the p2p libraries depend on syscall/js which doesn't work
on normal builds.
Also, clean up main.go to read a bit better.
* Update ho-http-js-libp2p to return errors from RoundTrip
* Add an LRU cache around the key DB
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.
Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
* breaking: Add Tracing.Enabled to toggle whether we do opentracing
Defaults to false, which is why this is a breaking change. We need
this flag because WASM builds cannot do opentracing.
* Start adding conditional builds for wasm to handle lib/pq
The general idea here is to have the wasm build have a `NewXXXDatabase`
that doesn't import any postgres package and hence we never import
`lib/pq`, which doesn't work under WASM (undefined `userCurrent`).
* Remove lib/pq for wasm for syncapi
* Add conditional building to remaining storage APIs
* Update build script to set env vars correctly for dendritejs
* sqlite bug fixes
* Docs
* Add a no-op main for dendritejs when not building under wasm
* Use the real prometheus, even for WASM
Instead, the dendrite-sw.js must mock out `process.pid` and
`fs.stat` - which must invoke the callback with an error (e.g `EINVAL`)
in order for it to work:
```
global.process = {
pid: 1,
};
global.fs.stat = function(path, cb) {
cb({
code: "EINVAL",
});
}
```
* Linting
2020-03-06 10:23:55 +00:00
|
|
|
latest_event_nids TEXT NOT NULL DEFAULT '[]',
|
2020-02-13 17:27:33 +00:00
|
|
|
last_event_sent_nid INTEGER NOT NULL DEFAULT 0,
|
|
|
|
state_snapshot_nid INTEGER NOT NULL DEFAULT 0,
|
2020-03-16 16:05:29 +00:00
|
|
|
room_version TEXT NOT NULL
|
2020-02-13 17:27:33 +00:00
|
|
|
);
|
|
|
|
`
|
|
|
|
|
|
|
|
// Same as insertEventTypeNIDSQL
|
|
|
|
const insertRoomNIDSQL = `
|
2020-03-16 16:05:29 +00:00
|
|
|
INSERT INTO roomserver_rooms (room_id, room_version) VALUES ($1, $2)
|
2022-03-17 18:24:27 +00:00
|
|
|
ON CONFLICT DO NOTHING
|
|
|
|
RETURNING room_nid;
|
2020-02-13 17:27:33 +00:00
|
|
|
`
|
|
|
|
|
|
|
|
const selectRoomNIDSQL = "" +
|
|
|
|
"SELECT room_nid FROM roomserver_rooms WHERE room_id = $1"
|
|
|
|
|
|
|
|
const selectLatestEventNIDsSQL = "" +
|
|
|
|
"SELECT latest_event_nids, state_snapshot_nid FROM roomserver_rooms WHERE room_nid = $1"
|
|
|
|
|
|
|
|
const selectLatestEventNIDsForUpdateSQL = "" +
|
|
|
|
"SELECT latest_event_nids, last_event_sent_nid, state_snapshot_nid FROM roomserver_rooms WHERE room_nid = $1"
|
|
|
|
|
|
|
|
const updateLatestEventNIDsSQL = "" +
|
|
|
|
"UPDATE roomserver_rooms SET latest_event_nids = $1, last_event_sent_nid = $2, state_snapshot_nid = $3 WHERE room_nid = $4"
|
|
|
|
|
2020-12-16 10:33:28 +00:00
|
|
|
const selectRoomVersionsForRoomNIDsSQL = "" +
|
|
|
|
"SELECT room_nid, room_version FROM roomserver_rooms WHERE room_nid IN ($1)"
|
2020-03-19 18:33:04 +00:00
|
|
|
|
2020-09-01 12:40:49 +01:00
|
|
|
const selectRoomInfoSQL = "" +
|
|
|
|
"SELECT room_version, room_nid, state_snapshot_nid, latest_event_nids FROM roomserver_rooms WHERE room_id = $1"
|
|
|
|
|
2020-09-03 17:20:54 +01:00
|
|
|
const bulkSelectRoomIDsSQL = "" +
|
|
|
|
"SELECT room_id FROM roomserver_rooms WHERE room_nid IN ($1)"
|
|
|
|
|
2020-09-03 18:27:02 +01:00
|
|
|
const bulkSelectRoomNIDsSQL = "" +
|
|
|
|
"SELECT room_nid FROM roomserver_rooms WHERE room_id IN ($1)"
|
|
|
|
|
2023-01-19 20:02:32 +00:00
|
|
|
const selectRoomNIDForUpdateSQL = "" +
|
|
|
|
"SELECT room_nid FROM roomserver_rooms WHERE room_id = $1"
|
|
|
|
|
2020-02-13 17:27:33 +00:00
|
|
|
type roomStatements struct {
|
2020-07-21 10:48:49 +01:00
|
|
|
db *sql.DB
|
2020-02-13 17:27:33 +00:00
|
|
|
insertRoomNIDStmt *sql.Stmt
|
|
|
|
selectRoomNIDStmt *sql.Stmt
|
2023-01-19 20:02:32 +00:00
|
|
|
selectRoomNIDForUpdateStmt *sql.Stmt
|
2020-02-13 17:27:33 +00:00
|
|
|
selectLatestEventNIDsStmt *sql.Stmt
|
|
|
|
selectLatestEventNIDsForUpdateStmt *sql.Stmt
|
|
|
|
updateLatestEventNIDsStmt *sql.Stmt
|
2020-12-16 10:33:28 +00:00
|
|
|
//selectRoomVersionForRoomNIDStmt *sql.Stmt
|
|
|
|
selectRoomInfoStmt *sql.Stmt
|
2020-02-13 17:27:33 +00:00
|
|
|
}
|
|
|
|
|
2022-05-16 18:33:16 +01:00
|
|
|
func CreateRoomsTable(db *sql.DB) error {
|
2021-04-26 13:25:57 +01:00
|
|
|
_, err := db.Exec(roomsSchema)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-05-16 18:33:16 +01:00
|
|
|
func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
|
2020-07-21 10:48:49 +01:00
|
|
|
s := &roomStatements{
|
2020-08-19 15:38:27 +01:00
|
|
|
db: db,
|
2020-07-21 10:48:49 +01:00
|
|
|
}
|
2021-04-26 13:25:57 +01:00
|
|
|
|
2021-07-28 18:30:04 +01:00
|
|
|
return s, sqlutil.StatementList{
|
2020-02-13 17:27:33 +00:00
|
|
|
{&s.insertRoomNIDStmt, insertRoomNIDSQL},
|
|
|
|
{&s.selectRoomNIDStmt, selectRoomNIDSQL},
|
|
|
|
{&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL},
|
|
|
|
{&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL},
|
|
|
|
{&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL},
|
2020-12-16 10:33:28 +00:00
|
|
|
//{&s.selectRoomVersionForRoomNIDsStmt, selectRoomVersionForRoomNIDsSQL},
|
2020-09-01 12:40:49 +01:00
|
|
|
{&s.selectRoomInfoStmt, selectRoomInfoSQL},
|
2023-01-19 20:02:32 +00:00
|
|
|
{&s.selectRoomNIDForUpdateStmt, selectRoomNIDForUpdateSQL},
|
2020-05-27 11:03:47 +01:00
|
|
|
}.Prepare(db)
|
2020-02-13 17:27:33 +00:00
|
|
|
}
|
|
|
|
|
2022-02-04 10:39:34 +00:00
|
|
|
func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
|
2020-09-01 12:40:49 +01:00
|
|
|
var info types.RoomInfo
|
|
|
|
var latestNIDsJSON string
|
2022-08-01 15:29:19 +01:00
|
|
|
var stateSnapshotNID types.StateSnapshotNID
|
2022-02-04 10:39:34 +00:00
|
|
|
stmt := sqlutil.TxStmt(txn, s.selectRoomInfoStmt)
|
|
|
|
err := stmt.QueryRowContext(ctx, roomID).Scan(
|
2022-08-01 15:29:19 +01:00
|
|
|
&info.RoomVersion, &info.RoomNID, &stateSnapshotNID, &latestNIDsJSON,
|
2020-09-01 12:40:49 +01:00
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
if err == sql.ErrNoRows {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
var latestNIDs []int64
|
|
|
|
if err = json.Unmarshal([]byte(latestNIDsJSON), &latestNIDs); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-08-01 15:29:19 +01:00
|
|
|
info.SetStateSnapshotNID(stateSnapshotNID)
|
|
|
|
info.SetIsStub(len(latestNIDs) == 0)
|
2020-09-01 12:40:49 +01:00
|
|
|
return &info, err
|
|
|
|
}
|
|
|
|
|
2020-05-26 18:23:39 +01:00
|
|
|
func (s *roomStatements) InsertRoomNID(
|
2020-03-16 16:05:29 +00:00
|
|
|
ctx context.Context, txn *sql.Tx,
|
|
|
|
roomID string, roomVersion gomatrixserverlib.RoomVersion,
|
2020-07-21 15:48:21 +01:00
|
|
|
) (roomNID types.RoomNID, err error) {
|
2020-08-19 15:38:27 +01:00
|
|
|
insertStmt := sqlutil.TxStmt(txn, s.insertRoomNIDStmt)
|
2022-03-17 18:24:27 +00:00
|
|
|
if err = insertStmt.QueryRowContext(ctx, roomID, roomVersion).Scan(&roomNID); err != nil {
|
|
|
|
return 0, fmt.Errorf("resultStmt.QueryRowContext.Scan: %w", err)
|
2020-02-13 17:27:33 +00:00
|
|
|
}
|
2020-07-21 15:48:21 +01:00
|
|
|
return
|
2020-02-13 17:27:33 +00:00
|
|
|
}
|
|
|
|
|
2020-05-26 18:23:39 +01:00
|
|
|
func (s *roomStatements) SelectRoomNID(
|
2020-02-13 17:27:33 +00:00
|
|
|
ctx context.Context, txn *sql.Tx, roomID string,
|
|
|
|
) (types.RoomNID, error) {
|
|
|
|
var roomNID int64
|
2020-06-12 14:55:57 +01:00
|
|
|
stmt := sqlutil.TxStmt(txn, s.selectRoomNIDStmt)
|
2020-02-13 17:27:33 +00:00
|
|
|
err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID)
|
|
|
|
return types.RoomNID(roomNID), err
|
|
|
|
}
|
|
|
|
|
2023-01-19 20:02:32 +00:00
|
|
|
func (s *roomStatements) SelectRoomNIDForUpdate(
|
|
|
|
ctx context.Context, txn *sql.Tx, roomID string,
|
|
|
|
) (types.RoomNID, error) {
|
|
|
|
var roomNID int64
|
|
|
|
stmt := sqlutil.TxStmt(txn, s.selectRoomNIDForUpdateStmt)
|
|
|
|
err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID)
|
|
|
|
return types.RoomNID(roomNID), err
|
|
|
|
}
|
|
|
|
|
2020-05-26 18:23:39 +01:00
|
|
|
func (s *roomStatements) SelectLatestEventNIDs(
|
2020-02-13 17:27:33 +00:00
|
|
|
ctx context.Context, txn *sql.Tx, roomNID types.RoomNID,
|
|
|
|
) ([]types.EventNID, types.StateSnapshotNID, error) {
|
Add peer-to-peer support into Dendrite via libp2p and fetch (#880)
* Use a fork of pq which supports userCurrent on wasm
* Use sqlite3_js driver when running in JS
* Add cmd/dendritejs to pull in sqlite3_js driver for wasm only
* Update to latest go-sqlite-js version
* Replace prometheus with a stub. sigh
* Hard-code a config and don't use opentracing
* Latest go-sqlite3-js version
* Generate a key for now
* Listen for fetch traffic rather than HTTP
* Latest hacks for js
* libp2p support
* More libp2p
* Fork gjson to allow us to enforce auth checks as before
Previously, all events would come down redacted because the hash
checks would fail. They would fail because sjson.DeleteBytes didn't
remove keys not used for hashing. This didn't work because of a build
tag which included a file which no-oped the index returned.
See https://github.com/tidwall/gjson/issues/157
When it's resolved, let's go back to mainline.
* Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157
* Use latest gomatrixserverlib for sig checks
* Fix a bug which could cause exclude_from_sync to not be set
Caused when sending events over federation.
* Use query variadic to make lookups actually work!
* Latest gomatrixserverlib
* Add notes on getting p2p up and running
Partly so I don't forget myself!
* refactor: Move p2p specific stuff to cmd/dendritejs
This is important or else the normal build of dendrite will fail
because the p2p libraries depend on syscall/js which doesn't work
on normal builds.
Also, clean up main.go to read a bit better.
* Update ho-http-js-libp2p to return errors from RoundTrip
* Add an LRU cache around the key DB
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.
Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
* breaking: Add Tracing.Enabled to toggle whether we do opentracing
Defaults to false, which is why this is a breaking change. We need
this flag because WASM builds cannot do opentracing.
* Start adding conditional builds for wasm to handle lib/pq
The general idea here is to have the wasm build have a `NewXXXDatabase`
that doesn't import any postgres package and hence we never import
`lib/pq`, which doesn't work under WASM (undefined `userCurrent`).
* Remove lib/pq for wasm for syncapi
* Add conditional building to remaining storage APIs
* Update build script to set env vars correctly for dendritejs
* sqlite bug fixes
* Docs
* Add a no-op main for dendritejs when not building under wasm
* Use the real prometheus, even for WASM
Instead, the dendrite-sw.js must mock out `process.pid` and
`fs.stat` - which must invoke the callback with an error (e.g `EINVAL`)
in order for it to work:
```
global.process = {
pid: 1,
};
global.fs.stat = function(path, cb) {
cb({
code: "EINVAL",
});
}
```
* Linting
2020-03-06 10:23:55 +00:00
|
|
|
var eventNIDs []types.EventNID
|
|
|
|
var nidsJSON string
|
2020-02-13 17:27:33 +00:00
|
|
|
var stateSnapshotNID int64
|
2020-06-12 14:55:57 +01:00
|
|
|
stmt := sqlutil.TxStmt(txn, s.selectLatestEventNIDsStmt)
|
Add peer-to-peer support into Dendrite via libp2p and fetch (#880)
* Use a fork of pq which supports userCurrent on wasm
* Use sqlite3_js driver when running in JS
* Add cmd/dendritejs to pull in sqlite3_js driver for wasm only
* Update to latest go-sqlite-js version
* Replace prometheus with a stub. sigh
* Hard-code a config and don't use opentracing
* Latest go-sqlite3-js version
* Generate a key for now
* Listen for fetch traffic rather than HTTP
* Latest hacks for js
* libp2p support
* More libp2p
* Fork gjson to allow us to enforce auth checks as before
Previously, all events would come down redacted because the hash
checks would fail. They would fail because sjson.DeleteBytes didn't
remove keys not used for hashing. This didn't work because of a build
tag which included a file which no-oped the index returned.
See https://github.com/tidwall/gjson/issues/157
When it's resolved, let's go back to mainline.
* Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157
* Use latest gomatrixserverlib for sig checks
* Fix a bug which could cause exclude_from_sync to not be set
Caused when sending events over federation.
* Use query variadic to make lookups actually work!
* Latest gomatrixserverlib
* Add notes on getting p2p up and running
Partly so I don't forget myself!
* refactor: Move p2p specific stuff to cmd/dendritejs
This is important or else the normal build of dendrite will fail
because the p2p libraries depend on syscall/js which doesn't work
on normal builds.
Also, clean up main.go to read a bit better.
* Update ho-http-js-libp2p to return errors from RoundTrip
* Add an LRU cache around the key DB
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.
Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
* breaking: Add Tracing.Enabled to toggle whether we do opentracing
Defaults to false, which is why this is a breaking change. We need
this flag because WASM builds cannot do opentracing.
* Start adding conditional builds for wasm to handle lib/pq
The general idea here is to have the wasm build have a `NewXXXDatabase`
that doesn't import any postgres package and hence we never import
`lib/pq`, which doesn't work under WASM (undefined `userCurrent`).
* Remove lib/pq for wasm for syncapi
* Add conditional building to remaining storage APIs
* Update build script to set env vars correctly for dendritejs
* sqlite bug fixes
* Docs
* Add a no-op main for dendritejs when not building under wasm
* Use the real prometheus, even for WASM
Instead, the dendrite-sw.js must mock out `process.pid` and
`fs.stat` - which must invoke the callback with an error (e.g `EINVAL`)
in order for it to work:
```
global.process = {
pid: 1,
};
global.fs.stat = function(path, cb) {
cb({
code: "EINVAL",
});
}
```
* Linting
2020-03-06 10:23:55 +00:00
|
|
|
err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &stateSnapshotNID)
|
2020-02-13 17:27:33 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, 0, err
|
|
|
|
}
|
Add peer-to-peer support into Dendrite via libp2p and fetch (#880)
* Use a fork of pq which supports userCurrent on wasm
* Use sqlite3_js driver when running in JS
* Add cmd/dendritejs to pull in sqlite3_js driver for wasm only
* Update to latest go-sqlite-js version
* Replace prometheus with a stub. sigh
* Hard-code a config and don't use opentracing
* Latest go-sqlite3-js version
* Generate a key for now
* Listen for fetch traffic rather than HTTP
* Latest hacks for js
* libp2p support
* More libp2p
* Fork gjson to allow us to enforce auth checks as before
Previously, all events would come down redacted because the hash
checks would fail. They would fail because sjson.DeleteBytes didn't
remove keys not used for hashing. This didn't work because of a build
tag which included a file which no-oped the index returned.
See https://github.com/tidwall/gjson/issues/157
When it's resolved, let's go back to mainline.
* Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157
* Use latest gomatrixserverlib for sig checks
* Fix a bug which could cause exclude_from_sync to not be set
Caused when sending events over federation.
* Use query variadic to make lookups actually work!
* Latest gomatrixserverlib
* Add notes on getting p2p up and running
Partly so I don't forget myself!
* refactor: Move p2p specific stuff to cmd/dendritejs
This is important or else the normal build of dendrite will fail
because the p2p libraries depend on syscall/js which doesn't work
on normal builds.
Also, clean up main.go to read a bit better.
* Update ho-http-js-libp2p to return errors from RoundTrip
* Add an LRU cache around the key DB
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.
Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
* breaking: Add Tracing.Enabled to toggle whether we do opentracing
Defaults to false, which is why this is a breaking change. We need
this flag because WASM builds cannot do opentracing.
* Start adding conditional builds for wasm to handle lib/pq
The general idea here is to have the wasm build have a `NewXXXDatabase`
that doesn't import any postgres package and hence we never import
`lib/pq`, which doesn't work under WASM (undefined `userCurrent`).
* Remove lib/pq for wasm for syncapi
* Add conditional building to remaining storage APIs
* Update build script to set env vars correctly for dendritejs
* sqlite bug fixes
* Docs
* Add a no-op main for dendritejs when not building under wasm
* Use the real prometheus, even for WASM
Instead, the dendrite-sw.js must mock out `process.pid` and
`fs.stat` - which must invoke the callback with an error (e.g `EINVAL`)
in order for it to work:
```
global.process = {
pid: 1,
};
global.fs.stat = function(path, cb) {
cb({
code: "EINVAL",
});
}
```
* Linting
2020-03-06 10:23:55 +00:00
|
|
|
if err := json.Unmarshal([]byte(nidsJSON), &eventNIDs); err != nil {
|
|
|
|
return nil, 0, err
|
2020-02-13 17:27:33 +00:00
|
|
|
}
|
|
|
|
return eventNIDs, types.StateSnapshotNID(stateSnapshotNID), nil
|
|
|
|
}
|
|
|
|
|
2020-05-26 18:23:39 +01:00
|
|
|
func (s *roomStatements) SelectLatestEventsNIDsForUpdate(
|
2020-02-13 17:27:33 +00:00
|
|
|
ctx context.Context, txn *sql.Tx, roomNID types.RoomNID,
|
|
|
|
) ([]types.EventNID, types.EventNID, types.StateSnapshotNID, error) {
|
Add peer-to-peer support into Dendrite via libp2p and fetch (#880)
* Use a fork of pq which supports userCurrent on wasm
* Use sqlite3_js driver when running in JS
* Add cmd/dendritejs to pull in sqlite3_js driver for wasm only
* Update to latest go-sqlite-js version
* Replace prometheus with a stub. sigh
* Hard-code a config and don't use opentracing
* Latest go-sqlite3-js version
* Generate a key for now
* Listen for fetch traffic rather than HTTP
* Latest hacks for js
* libp2p support
* More libp2p
* Fork gjson to allow us to enforce auth checks as before
Previously, all events would come down redacted because the hash
checks would fail. They would fail because sjson.DeleteBytes didn't
remove keys not used for hashing. This didn't work because of a build
tag which included a file which no-oped the index returned.
See https://github.com/tidwall/gjson/issues/157
When it's resolved, let's go back to mainline.
* Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157
* Use latest gomatrixserverlib for sig checks
* Fix a bug which could cause exclude_from_sync to not be set
Caused when sending events over federation.
* Use query variadic to make lookups actually work!
* Latest gomatrixserverlib
* Add notes on getting p2p up and running
Partly so I don't forget myself!
* refactor: Move p2p specific stuff to cmd/dendritejs
This is important or else the normal build of dendrite will fail
because the p2p libraries depend on syscall/js which doesn't work
on normal builds.
Also, clean up main.go to read a bit better.
* Update ho-http-js-libp2p to return errors from RoundTrip
* Add an LRU cache around the key DB
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.
Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
* breaking: Add Tracing.Enabled to toggle whether we do opentracing
Defaults to false, which is why this is a breaking change. We need
this flag because WASM builds cannot do opentracing.
* Start adding conditional builds for wasm to handle lib/pq
The general idea here is to have the wasm build have a `NewXXXDatabase`
that doesn't import any postgres package and hence we never import
`lib/pq`, which doesn't work under WASM (undefined `userCurrent`).
* Remove lib/pq for wasm for syncapi
* Add conditional building to remaining storage APIs
* Update build script to set env vars correctly for dendritejs
* sqlite bug fixes
* Docs
* Add a no-op main for dendritejs when not building under wasm
* Use the real prometheus, even for WASM
Instead, the dendrite-sw.js must mock out `process.pid` and
`fs.stat` - which must invoke the callback with an error (e.g `EINVAL`)
in order for it to work:
```
global.process = {
pid: 1,
};
global.fs.stat = function(path, cb) {
cb({
code: "EINVAL",
});
}
```
* Linting
2020-03-06 10:23:55 +00:00
|
|
|
var eventNIDs []types.EventNID
|
|
|
|
var nidsJSON string
|
2020-02-13 17:27:33 +00:00
|
|
|
var lastEventSentNID int64
|
|
|
|
var stateSnapshotNID int64
|
2020-06-12 14:55:57 +01:00
|
|
|
stmt := sqlutil.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt)
|
Add peer-to-peer support into Dendrite via libp2p and fetch (#880)
* Use a fork of pq which supports userCurrent on wasm
* Use sqlite3_js driver when running in JS
* Add cmd/dendritejs to pull in sqlite3_js driver for wasm only
* Update to latest go-sqlite-js version
* Replace prometheus with a stub. sigh
* Hard-code a config and don't use opentracing
* Latest go-sqlite3-js version
* Generate a key for now
* Listen for fetch traffic rather than HTTP
* Latest hacks for js
* libp2p support
* More libp2p
* Fork gjson to allow us to enforce auth checks as before
Previously, all events would come down redacted because the hash
checks would fail. They would fail because sjson.DeleteBytes didn't
remove keys not used for hashing. This didn't work because of a build
tag which included a file which no-oped the index returned.
See https://github.com/tidwall/gjson/issues/157
When it's resolved, let's go back to mainline.
* Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157
* Use latest gomatrixserverlib for sig checks
* Fix a bug which could cause exclude_from_sync to not be set
Caused when sending events over federation.
* Use query variadic to make lookups actually work!
* Latest gomatrixserverlib
* Add notes on getting p2p up and running
Partly so I don't forget myself!
* refactor: Move p2p specific stuff to cmd/dendritejs
This is important or else the normal build of dendrite will fail
because the p2p libraries depend on syscall/js which doesn't work
on normal builds.
Also, clean up main.go to read a bit better.
* Update ho-http-js-libp2p to return errors from RoundTrip
* Add an LRU cache around the key DB
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.
Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
* breaking: Add Tracing.Enabled to toggle whether we do opentracing
Defaults to false, which is why this is a breaking change. We need
this flag because WASM builds cannot do opentracing.
* Start adding conditional builds for wasm to handle lib/pq
The general idea here is to have the wasm build have a `NewXXXDatabase`
that doesn't import any postgres package and hence we never import
`lib/pq`, which doesn't work under WASM (undefined `userCurrent`).
* Remove lib/pq for wasm for syncapi
* Add conditional building to remaining storage APIs
* Update build script to set env vars correctly for dendritejs
* sqlite bug fixes
* Docs
* Add a no-op main for dendritejs when not building under wasm
* Use the real prometheus, even for WASM
Instead, the dendrite-sw.js must mock out `process.pid` and
`fs.stat` - which must invoke the callback with an error (e.g `EINVAL`)
in order for it to work:
```
global.process = {
pid: 1,
};
global.fs.stat = function(path, cb) {
cb({
code: "EINVAL",
});
}
```
* Linting
2020-03-06 10:23:55 +00:00
|
|
|
err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &lastEventSentNID, &stateSnapshotNID)
|
2020-02-13 17:27:33 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, 0, 0, err
|
|
|
|
}
|
Add peer-to-peer support into Dendrite via libp2p and fetch (#880)
* Use a fork of pq which supports userCurrent on wasm
* Use sqlite3_js driver when running in JS
* Add cmd/dendritejs to pull in sqlite3_js driver for wasm only
* Update to latest go-sqlite-js version
* Replace prometheus with a stub. sigh
* Hard-code a config and don't use opentracing
* Latest go-sqlite3-js version
* Generate a key for now
* Listen for fetch traffic rather than HTTP
* Latest hacks for js
* libp2p support
* More libp2p
* Fork gjson to allow us to enforce auth checks as before
Previously, all events would come down redacted because the hash
checks would fail. They would fail because sjson.DeleteBytes didn't
remove keys not used for hashing. This didn't work because of a build
tag which included a file which no-oped the index returned.
See https://github.com/tidwall/gjson/issues/157
When it's resolved, let's go back to mainline.
* Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157
* Use latest gomatrixserverlib for sig checks
* Fix a bug which could cause exclude_from_sync to not be set
Caused when sending events over federation.
* Use query variadic to make lookups actually work!
* Latest gomatrixserverlib
* Add notes on getting p2p up and running
Partly so I don't forget myself!
* refactor: Move p2p specific stuff to cmd/dendritejs
This is important or else the normal build of dendrite will fail
because the p2p libraries depend on syscall/js which doesn't work
on normal builds.
Also, clean up main.go to read a bit better.
* Update ho-http-js-libp2p to return errors from RoundTrip
* Add an LRU cache around the key DB
We actually need this for P2P because otherwise we can *segfault*
with things like: "runtime: unexpected return pc for runtime.handleEvent"
where the event is a `syscall/js` event, caused by spamming sql.js
caused by "Checking event signatures for 14 events of room state" which
hammers the key DB repeatedly in quick succession.
Using a cache fixes this, though the underlying cause is probably a bug
in the version of Go I'm on (1.13.7)
* breaking: Add Tracing.Enabled to toggle whether we do opentracing
Defaults to false, which is why this is a breaking change. We need
this flag because WASM builds cannot do opentracing.
* Start adding conditional builds for wasm to handle lib/pq
The general idea here is to have the wasm build have a `NewXXXDatabase`
that doesn't import any postgres package and hence we never import
`lib/pq`, which doesn't work under WASM (undefined `userCurrent`).
* Remove lib/pq for wasm for syncapi
* Add conditional building to remaining storage APIs
* Update build script to set env vars correctly for dendritejs
* sqlite bug fixes
* Docs
* Add a no-op main for dendritejs when not building under wasm
* Use the real prometheus, even for WASM
Instead, the dendrite-sw.js must mock out `process.pid` and
`fs.stat` - which must invoke the callback with an error (e.g `EINVAL`)
in order for it to work:
```
global.process = {
pid: 1,
};
global.fs.stat = function(path, cb) {
cb({
code: "EINVAL",
});
}
```
* Linting
2020-03-06 10:23:55 +00:00
|
|
|
if err := json.Unmarshal([]byte(nidsJSON), &eventNIDs); err != nil {
|
|
|
|
return nil, 0, 0, err
|
2020-02-13 17:27:33 +00:00
|
|
|
}
|
|
|
|
return eventNIDs, types.EventNID(lastEventSentNID), types.StateSnapshotNID(stateSnapshotNID), nil
|
|
|
|
}
|
|
|
|
|
2020-05-26 18:23:39 +01:00
|
|
|
func (s *roomStatements) UpdateLatestEventNIDs(
|
2020-02-13 17:27:33 +00:00
|
|
|
ctx context.Context,
|
|
|
|
txn *sql.Tx,
|
|
|
|
roomNID types.RoomNID,
|
|
|
|
eventNIDs []types.EventNID,
|
|
|
|
lastEventSentNID types.EventNID,
|
|
|
|
stateSnapshotNID types.StateSnapshotNID,
|
|
|
|
) error {
|
2020-08-19 15:38:27 +01:00
|
|
|
stmt := sqlutil.TxStmt(txn, s.updateLatestEventNIDsStmt)
|
|
|
|
_, err := stmt.ExecContext(
|
|
|
|
ctx,
|
|
|
|
eventNIDsAsArray(eventNIDs),
|
|
|
|
int64(lastEventSentNID),
|
|
|
|
int64(stateSnapshotNID),
|
|
|
|
roomNID,
|
|
|
|
)
|
|
|
|
return err
|
2020-02-13 17:27:33 +00:00
|
|
|
}
|
|
|
|
|
2020-12-16 10:33:28 +00:00
|
|
|
func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
|
2022-02-04 10:39:34 +00:00
|
|
|
ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID,
|
2020-12-16 10:33:28 +00:00
|
|
|
) (map[types.RoomNID]gomatrixserverlib.RoomVersion, error) {
|
|
|
|
sqlStr := strings.Replace(selectRoomVersionsForRoomNIDsSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1)
|
|
|
|
sqlPrep, err := s.db.Prepare(sqlStr)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-03-04 15:05:42 +00:00
|
|
|
defer sqlPrep.Close() // nolint:errcheck
|
|
|
|
sqlStmt := sqlutil.TxStmt(txn, sqlPrep)
|
2020-12-16 10:33:28 +00:00
|
|
|
iRoomNIDs := make([]interface{}, len(roomNIDs))
|
|
|
|
for i, v := range roomNIDs {
|
|
|
|
iRoomNIDs[i] = v
|
|
|
|
}
|
2022-03-04 15:05:42 +00:00
|
|
|
rows, err := sqlStmt.QueryContext(ctx, iRoomNIDs...)
|
2020-12-16 10:33:28 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed")
|
|
|
|
result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion)
|
2022-05-16 18:33:16 +01:00
|
|
|
var roomNID types.RoomNID
|
|
|
|
var roomVersion gomatrixserverlib.RoomVersion
|
2020-12-16 10:33:28 +00:00
|
|
|
for rows.Next() {
|
|
|
|
if err = rows.Scan(&roomNID, &roomVersion); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
result[roomNID] = roomVersion
|
2020-03-27 16:28:22 +00:00
|
|
|
}
|
2023-11-09 07:42:33 +00:00
|
|
|
return result, rows.Err()
|
2020-03-19 18:33:04 +00:00
|
|
|
}
|
2020-09-03 17:20:54 +01:00
|
|
|
|
2022-02-04 10:39:34 +00:00
|
|
|
func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) {
|
2020-09-03 17:20:54 +01:00
|
|
|
iRoomNIDs := make([]interface{}, len(roomNIDs))
|
|
|
|
for i, v := range roomNIDs {
|
|
|
|
iRoomNIDs[i] = v
|
|
|
|
}
|
|
|
|
sqlQuery := strings.Replace(bulkSelectRoomIDsSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1)
|
2022-02-04 10:39:34 +00:00
|
|
|
var rows *sql.Rows
|
|
|
|
var err error
|
|
|
|
if txn != nil {
|
|
|
|
rows, err = txn.QueryContext(ctx, sqlQuery, iRoomNIDs...)
|
|
|
|
} else {
|
|
|
|
rows, err = s.db.QueryContext(ctx, sqlQuery, iRoomNIDs...)
|
|
|
|
}
|
2020-09-03 17:20:54 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed")
|
|
|
|
var roomIDs []string
|
2022-05-16 18:33:16 +01:00
|
|
|
var roomID string
|
2020-09-03 17:20:54 +01:00
|
|
|
for rows.Next() {
|
|
|
|
if err = rows.Scan(&roomID); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
roomIDs = append(roomIDs, roomID)
|
|
|
|
}
|
2023-11-09 07:42:33 +00:00
|
|
|
return roomIDs, rows.Err()
|
2020-09-03 17:20:54 +01:00
|
|
|
}
|
2020-09-03 18:27:02 +01:00
|
|
|
|
2022-02-04 10:39:34 +00:00
|
|
|
func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) {
|
2020-09-03 18:27:02 +01:00
|
|
|
iRoomIDs := make([]interface{}, len(roomIDs))
|
|
|
|
for i, v := range roomIDs {
|
|
|
|
iRoomIDs[i] = v
|
|
|
|
}
|
|
|
|
sqlQuery := strings.Replace(bulkSelectRoomNIDsSQL, "($1)", sqlutil.QueryVariadic(len(roomIDs)), 1)
|
2022-02-04 10:39:34 +00:00
|
|
|
var rows *sql.Rows
|
|
|
|
var err error
|
|
|
|
if txn != nil {
|
|
|
|
rows, err = txn.QueryContext(ctx, sqlQuery, iRoomIDs...)
|
|
|
|
} else {
|
|
|
|
rows, err = s.db.QueryContext(ctx, sqlQuery, iRoomIDs...)
|
|
|
|
}
|
2020-09-03 18:27:02 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed")
|
|
|
|
var roomNIDs []types.RoomNID
|
2022-05-16 18:33:16 +01:00
|
|
|
var roomNID types.RoomNID
|
2020-09-03 18:27:02 +01:00
|
|
|
for rows.Next() {
|
|
|
|
if err = rows.Scan(&roomNID); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
roomNIDs = append(roomNIDs, roomNID)
|
|
|
|
}
|
2023-11-09 07:42:33 +00:00
|
|
|
return roomNIDs, rows.Err()
|
2020-09-03 18:27:02 +01:00
|
|
|
}
|