Bind build support, further Yggdrasil demo updates (#1152)

* Add gobind builds for Yggdrasil demo

* Massage client API a bit

* Fix build

* Fix gobind build

* Fix gobind client API setup

* Tweaks

* Tweaks

* Update sytest-whitelist, add comment

* Default to sending push rules on initial sync

build/gobind/build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+gomobile bind -v \
+    -ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \
+    -target ios \
+    github.com/matrix-org/dendrite/build/gobind

build/gobind/monolith.go

@@ -0,0 +1,161 @@
+package gobind
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+type DendriteMonolith struct {
+	StorageDirectory string
+	listener         net.Listener
+}
+
+func (m *DendriteMonolith) BaseURL() string {
+	return fmt.Sprintf("http://%s", m.listener.Addr().String())
+}
+
+func (m *DendriteMonolith) Start() {
+	logger := logrus.Logger{
+		Out: BindLogger{},
+	}
+	logrus.SetOutput(BindLogger{})
+
+	var err error
+	m.listener, err = net.Listen("tcp", "localhost:65432")
+	if err != nil {
+		panic(err)
+	}
+
+	ygg, err := yggconn.Setup("dendrite", "", m.StorageDirectory)
+	if err != nil {
+		panic(err)
+	}
+
+	cfg := &config.Dendrite{}
+	cfg.SetDefaults()
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
+	cfg.Matrix.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Kafka.UseNaffka = true
+	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
+	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory))
+	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory))
+	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory))
+	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory))
+	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory))
+	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory))
+	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory))
+	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory))
+	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory))
+	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory))
+	if err = cfg.Derive(); err != nil {
+		panic(err)
+	}
+
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	deviceDB := base.CreateDeviceDB()
+	federation := ygg.CreateFederationClient(base)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices)
+
+	rsAPI := roomserver.NewInternalAPI(
+		base, keyRing, federation,
+	)
+
+	eduInputAPI := eduserver.NewInternalAPI(
+		base, cache.New(), userAPI,
+	)
+
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing,
+	)
+
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this dependency
+	rsAPI.SetFederationSenderAPI(fsAPI)
+
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to public rooms db")
+	}
+
+	monolith := setup.Monolith{
+		Config:        base.Cfg,
+		AccountDB:     accountDB,
+		DeviceDB:      deviceDB,
+		Client:        ygg.CreateClient(base),
+		FedClient:     federation,
+		KeyRing:       keyRing,
+		KafkaConsumer: base.KafkaConsumer,
+		KafkaProducer: base.KafkaProducer,
+
+		AppserviceAPI:       asAPI,
+		EDUInternalAPI:      eduInputAPI,
+		FederationSenderAPI: fsAPI,
+		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
+		//ServerKeyAPI:        serverKeyAPI,
+
+		PublicRoomsDB: publicRoomsDB,
+	}
+	monolith.AddAllPublicRoutes(base.PublicAPIMux)
+
+	httputil.SetupHTTPAPI(
+		base.BaseMux,
+		base.PublicAPIMux,
+		base.InternalAPIMux,
+		cfg,
+		base.UseHTTPAPIs,
+	)
+
+	// Build both ends of a HTTP multiplex.
+	httpServer := &http.Server{
+		Addr:         ":0",
+		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+		ReadTimeout:  15 * time.Second,
+		WriteTimeout: 45 * time.Second,
+		IdleTimeout:  60 * time.Second,
+		BaseContext: func(_ net.Listener) context.Context {
+			return context.Background()
+		},
+		Handler: base.BaseMux,
+	}
+
+	go func() {
+		logger.Info("Listening on ", ygg.DerivedServerName())
+		logger.Fatal(httpServer.Serve(ygg))
+	}()
+	go func() {
+		logger.Info("Listening on ", m.BaseURL())
+		logger.Fatal(httpServer.Serve(m.listener))
+	}()
+}

build/gobind/platform_ios.go

@@ -0,0 +1,25 @@
+// +build ios
+
+package gobind
+
+/*
+#cgo CFLAGS: -x objective-c
+#cgo LDFLAGS: -framework Foundation
+#import <Foundation/Foundation.h>
+void Log(const char *text) {
+  NSString *nss = [NSString stringWithUTF8String:text];
+  NSLog(@"%@", nss);
+}
+*/
+import "C"
+import "unsafe"
+
+type BindLogger struct {
+}
+
+func (nsl BindLogger) Write(p []byte) (n int, err error) {
+	p = append(p, 0)
+	cstr := (*C.char)(unsafe.Pointer(&p[0]))
+	C.Log(cstr)
+	return len(p), nil
+}

build/gobind/platform_other.go

@@ -0,0 +1,12 @@
+// +build !ios
+
+package gobind
+
+import "log"
+
+type BindLogger struct{}
+
+func (nsl BindLogger) Write(p []byte) (n int, err error) {
+	log.Println(string(p))
+	return len(p), nil
+}

clientapi/routing/device.go

@@ -26,9 +26,12 @@ import (
 	"github.com/matrix-org/util"
 )
 
+// https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-devices
 type deviceJSON struct {
-	DeviceID string `json:"device_id"`
+	DeviceID    string `json:"device_id"`
-	UserID   string `json:"user_id"`
+	DisplayName string `json:"display_name"`
+	LastSeenIP  string `json:"last_seen_ip"`
+	LastSeenTS  uint64 `json:"last_seen_ts"`
 }
 
 type devicesJSON struct {
@@ -70,7 +73,6 @@ func GetDeviceByID(
 		Code: http.StatusOK,
 		JSON: deviceJSON{
 			DeviceID: dev.ID,
-			UserID:   dev.UserID,
 		},
 	}
 }
@@ -98,7 +100,6 @@ func GetDevicesByLocalpart(
 	for _, dev := range deviceList {
 		res.Devices = append(res.Devices, deviceJSON{
 			DeviceID: dev.ID,
-			UserID:   dev.UserID,
 		})
 	}
 

clientapi/routing/joinroom.go

@@ -37,6 +37,7 @@ func JoinRoomByIDOrAlias(
 	joinReq := roomserverAPI.PerformJoinRequest{
 		RoomIDOrAlias: roomIDOrAlias,
 		UserID:        device.UserID,
+		Content:       map[string]interface{}{},
 	}
 	joinRes := roomserverAPI.PerformJoinResponse{}
 

cmd/dendrite-demo-yggdrasil/main.go

@@ -47,52 +47,11 @@ var (
 	instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to")
 )
 
-type yggroundtripper struct {
-	inner *http.Transport
-}
-
-func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.URL.Scheme = "http"
-	return y.inner.RoundTrip(req)
-}
-
-func createFederationClient(
-	base *setup.BaseDendrite, n *yggconn.Node,
-) *gomatrixserverlib.FederationClient {
-	tr := &http.Transport{}
-	tr.RegisterProtocol(
-		"matrix", &yggroundtripper{
-			inner: &http.Transport{
-				ResponseHeaderTimeout: 15 * time.Second,
-				IdleConnTimeout:       60 * time.Second,
-				DialContext:           n.DialerContext,
-			},
-		},
-	)
-	return gomatrixserverlib.NewFederationClientWithTransport(
-		base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr,
-	)
-}
-
-func createClient(n *yggconn.Node) *gomatrixserverlib.Client {
-	tr := &http.Transport{}
-	tr.RegisterProtocol(
-		"matrix", &yggroundtripper{
-			inner: &http.Transport{
-				ResponseHeaderTimeout: 15 * time.Second,
-				IdleConnTimeout:       60 * time.Second,
-				DialContext:           n.DialerContext,
-			},
-		},
-	)
-	return gomatrixserverlib.NewClientWithTransport(tr)
-}
-
 // nolint:gocyclo
 func main() {
 	flag.Parse()
 
-	ygg, err := yggconn.Setup(*instanceName, *instancePeer)
+	ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".")
 	if err != nil {
 		panic(err)
 	}
@@ -125,7 +84,7 @@ func main() {
 
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
-	federation := createFederationClient(base, ygg)
+	federation := ygg.CreateFederationClient(base)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
@@ -160,7 +119,7 @@ func main() {
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
-		Client:        createClient(ygg),
+		Client:        ygg.CreateClient(base),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -203,7 +162,7 @@ func main() {
 		logrus.Fatal(httpServer.Serve(ygg))
 	}()
 	go func() {
-		httpBindAddr := fmt.Sprintf("localhost:%d", *instancePort)
+		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
 		logrus.Info("Listening on ", httpBindAddr)
 		logrus.Fatal(http.ListenAndServe(httpBindAddr, base.BaseMux))
 	}()

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -0,0 +1,74 @@
+package yggconn
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func (n *Node) yggdialer(_, address string) (net.Conn, error) {
+	tokens := strings.Split(address, ":")
+	raw, err := hex.DecodeString(tokens[0])
+	if err != nil {
+		return nil, fmt.Errorf("hex.DecodeString: %w", err)
+	}
+	converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw))
+	convhex := hex.EncodeToString(converted)
+	return n.Dial("curve25519", convhex)
+}
+
+func (n *Node) yggdialerctx(ctx context.Context, network, address string) (net.Conn, error) {
+	return n.yggdialer(network, address)
+}
+
+type yggroundtripper struct {
+	inner *http.Transport
+}
+
+func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = "http"
+	return y.inner.RoundTrip(req)
+}
+
+func (n *Node) CreateClient(
+	base *setup.BaseDendrite,
+) *gomatrixserverlib.Client {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix", &yggroundtripper{
+			inner: &http.Transport{
+				ResponseHeaderTimeout: 15 * time.Second,
+				IdleConnTimeout:       60 * time.Second,
+				DialContext:           n.yggdialerctx,
+			},
+		},
+	)
+	return gomatrixserverlib.NewClientWithTransport(tr)
+}
+
+func (n *Node) CreateFederationClient(
+	base *setup.BaseDendrite,
+) *gomatrixserverlib.FederationClient {
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix", &yggroundtripper{
+			inner: &http.Transport{
+				ResponseHeaderTimeout: 15 * time.Second,
+				IdleConnTimeout:       60 * time.Second,
+				DialContext:           n.yggdialerctx,
+			},
+		},
+	)
+	return gomatrixserverlib.NewFederationClientWithTransport(
+		base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr,
+	)
+}

cmd/dendrite-demo-yggdrasil/yggconn/node.go

@@ -67,7 +67,7 @@ func (n *Node) DialerContext(ctx context.Context, network, address string) (net.
 }
 
 // nolint:gocyclo
-func Setup(instanceName, instancePeer string) (*Node, error) {
+func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) {
 	n := &Node{
 		core:      &yggdrasil.Core{},
 		config:    yggdrasilconfig.GenerateConfig(),
@@ -77,7 +77,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) {
 		incoming:  make(chan *yamux.Stream),
 	}
 
-	yggfile := fmt.Sprintf("%s-yggdrasil.conf", instanceName)
+	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
 	if _, err := os.Stat(yggfile); !os.IsNotExist(err) {
 		yggconf, e := ioutil.ReadFile(yggfile)
 		if e != nil {
@@ -87,7 +87,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) {
 			panic(err)
 		}
 	} else {
-		n.config.AdminListen = fmt.Sprintf("unix://./%s-yggdrasil.sock", instanceName)
+		n.config.AdminListen = "none" // fmt.Sprintf("unix://%s/%s-yggdrasil.sock", storageDirectory, instanceName)
 		n.config.MulticastInterfaces = []string{".*"}
 		n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey())
 		n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey())
@@ -114,20 +114,22 @@ func Setup(instanceName, instancePeer string) (*Node, error) {
 			panic(err)
 		}
 	}
-	if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil {
+	/*
-		panic(err)
+		if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil {
-	}
+			panic(err)
-	if err = n.admin.Start(); err != nil {
+		}
-		panic(err)
+		if err = n.admin.Start(); err != nil {
-	}
+			panic(err)
+		}
+	*/
 	if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil {
 		panic(err)
 	}
 	if err = n.multicast.Start(); err != nil {
 		panic(err)
 	}
-	n.admin.SetupAdminHandlers(n.admin)
+	//n.admin.SetupAdminHandlers(n.admin)
-	n.multicast.SetupAdminHandlers(n.admin)
+	//n.multicast.SetupAdminHandlers(n.admin)
 	n.listener, err = n.core.ConnListen()
 	if err != nil {
 		panic(err)
@@ -137,6 +139,9 @@ func Setup(instanceName, instancePeer string) (*Node, error) {
 		panic(err)
 	}
 
+	n.log.Println("Public curve25519:", n.core.EncryptionPublicKey())
+	n.log.Println("Public ed25519:", n.core.SigningPublicKey())
+
 	go n.listenFromYgg()
 
 	return n, nil

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b
 	go.uber.org/atomic v1.4.0
 	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
+	golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 // indirect
 	gopkg.in/h2non/bimg.v1 v1.0.18
 	gopkg.in/yaml.v2 v2.2.8
 )

go.sum

@@ -4,6 +4,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv
 github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48=
 github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
 github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA=
 github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
@@ -589,6 +590,7 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -599,9 +601,18 @@ golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 h1:JxsyO7zPDWn1rBZW8FV5RFwCKqYeXnyaS/VQPLpXu6I=
+golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -671,6 +682,10 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk=
+golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

syncapi/sync/request.go

@@ -62,6 +62,10 @@ func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, err
 		}
 		since = &tok
 	}
+	if since == nil {
+		tok := types.NewStreamToken(0, 0)
+		since = &tok
+	}
 	timelineLimit := defaultTimelineLimit
 	// TODO: read from stored filters too
 	filterQuery := req.URL.Query().Get("filter")

syncapi/sync/requestpool.go

@@ -256,7 +256,8 @@ func (rp *RequestPool) appendAccountData(
 	}
 
 	if len(dataTypes) == 0 {
-		return data, nil
+		// TODO: this fixes the sytest but is it the right thing to do?
+		dataTypes[""] = []string{"m.push_rules"}
 	}
 
 	// Iterate over the rooms

syncapi/types/types.go

@@ -98,6 +98,9 @@ func (t *StreamingToken) PDUPosition() StreamPosition {
 func (t *StreamingToken) EDUPosition() StreamPosition {
 	return t.Positions[1]
 }
+func (t *StreamingToken) String() string {
+	return t.syncToken.String()
+}
 
 // IsAfter returns true if ANY position in this token is greater than `other`.
 func (t *StreamingToken) IsAfter(other StreamingToken) bool {
@@ -220,8 +223,8 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) {
 		err = fmt.Errorf("token %s is not a topology token", tok)
 		return
 	}
-	if len(t.Positions) != 2 {
+	if len(t.Positions) < 2 {
-		err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions))
+		err = fmt.Errorf("token %s wrong number of values, got %d want at least 2", tok, len(t.Positions))
 		return
 	}
 	return TopologyToken{
@@ -247,8 +250,8 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 		err = fmt.Errorf("token %s is not a streaming token", tok)
 		return
 	}
-	if len(t.Positions) != 2 {
+	if len(t.Positions) < 2 {
-		err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions))
+		err = fmt.Errorf("token %s wrong number of values, got %d want at least 2", tok, len(t.Positions))
 		return
 	}
 	return StreamingToken{

sytest-whitelist

@@ -347,3 +347,6 @@ GET /rooms/:room_id/state/m.room.topic gets topic
 GET /rooms/:room_id/state fetches entire room state
 Setting room topic reports m.room.topic to myself
 setting 'm.room.name' respects room powerlevel
+Syncing a new room with a large timeline limit isn't limited
+Left rooms appear in the leave section of sync
+Banned rooms appear in the leave section of sync