Update Pinecone P2P demo

This commit is contained in:
Neil Alexander 2022-03-30 15:01:22 +01:00
parent 49dc49b232
commit 8213b2ba30
No known key found for this signature in database
GPG Key ID: A02A2019A2BB0944
13 changed files with 157 additions and 14 deletions

View File

@ -1,3 +1,17 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gobind package gobind
import ( import (
@ -353,6 +367,7 @@ func (m *DendriteMonolith) Start() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
pMux := mux.NewRouter().SkipClean(true).UseEncodedPath() pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles) pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)

View File

@ -1,3 +1,17 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build ios //go:build ios
// +build ios // +build ios

View File

@ -1,3 +1,17 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !ios //go:build !ios
// +build !ios // +build !ios

View File

@ -1,3 +1,17 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package conn package conn
import ( import (

View File

@ -1,3 +1,17 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package conn package conn
import ( import (

View File

@ -0,0 +1,21 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package defaults
import "github.com/matrix-org/gomatrixserverlib"
var DefaultServerNames = map[gomatrixserverlib.ServerName]struct{}{
"3bf0258d23c60952639cc4c69c71d1508a7d43a0475d9000ff900a1848411ec7": {},
}

View File

@ -1,3 +1,17 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build elementweb //go:build elementweb
// +build elementweb // +build elementweb

View File

@ -1,3 +1,17 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !elementweb //go:build !elementweb
// +build !elementweb // +build !elementweb

View File

@ -1,4 +1,4 @@
// Copyright 2020 The Matrix.org Foundation C.I.C. // Copyright 2022 The Matrix.org Foundation C.I.C.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -245,6 +245,7 @@ func main() {
logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch") logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
} }
}) })
httpRouter.HandleFunc("/pinecone", pRouter.ManholeHandler)
embed.Embed(httpRouter, *instancePort, "Pinecone Demo") embed.Embed(httpRouter, *instancePort, "Pinecone Demo")
pMux := mux.NewRouter().SkipClean(true).UseEncodedPath() pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()

View File

@ -1,4 +1,4 @@
// Copyright 2020 The Matrix.org Foundation C.I.C. // Copyright 2022 The Matrix.org Foundation C.I.C.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -19,6 +19,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults"
"github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util" "github.com/matrix-org/util"
@ -50,9 +51,12 @@ func NewPineconeRoomProvider(
} }
func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
list := []gomatrixserverlib.ServerName{} list := map[gomatrixserverlib.ServerName]struct{}{}
for k := range defaults.DefaultServerNames {
list[k] = struct{}{}
}
for _, k := range p.r.Peers() { for _, k := range p.r.Peers() {
list = append(list, gomatrixserverlib.ServerName(k.PublicKey)) list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
} }
return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list) return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list)
} }
@ -61,7 +65,7 @@ func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
// Returns a list of public rooms. // Returns a list of public rooms.
func bulkFetchPublicRoomsFromServers( func bulkFetchPublicRoomsFromServers(
ctx context.Context, fedClient *gomatrixserverlib.FederationClient, ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
homeservers []gomatrixserverlib.ServerName, homeservers map[gomatrixserverlib.ServerName]struct{},
) (publicRooms []gomatrixserverlib.PublicRoom) { ) (publicRooms []gomatrixserverlib.PublicRoom) {
limit := 200 limit := 200
// follow pipeline semantics, see https://blog.golang.org/pipelines for more info. // follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
@ -74,7 +78,7 @@ func bulkFetchPublicRoomsFromServers(
wg.Add(len(homeservers)) wg.Add(len(homeservers))
// concurrently query for public rooms // concurrently query for public rooms
reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5) reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
for _, hs := range homeservers { for hs := range homeservers {
go func(homeserverDomain gomatrixserverlib.ServerName) { go func(homeserverDomain gomatrixserverlib.ServerName) {
defer wg.Done() defer wg.Done()
util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms") util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")

View File

@ -1,3 +1,17 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package users package users
import ( import (
@ -11,6 +25,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/authtypes"
clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil" clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults"
userapi "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util" "github.com/matrix-org/util"
@ -64,9 +79,12 @@ func (p *PineconeUserProvider) FederatedUserProfiles(w http.ResponseWriter, r *h
} }
func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error { func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error {
list := map[string]struct{}{} list := map[gomatrixserverlib.ServerName]struct{}{}
for k := range defaults.DefaultServerNames {
list[k] = struct{}{}
}
for _, k := range p.r.Peers() { for _, k := range p.r.Peers() {
list[k.PublicKey] = struct{}{} list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
} }
res.Profiles = bulkFetchUserDirectoriesFromServers(context.Background(), req, p.fedClient, list) res.Profiles = bulkFetchUserDirectoriesFromServers(context.Background(), req, p.fedClient, list)
return nil return nil
@ -77,7 +95,7 @@ func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *use
func bulkFetchUserDirectoriesFromServers( func bulkFetchUserDirectoriesFromServers(
ctx context.Context, req *userapi.QuerySearchProfilesRequest, ctx context.Context, req *userapi.QuerySearchProfilesRequest,
fedClient *gomatrixserverlib.FederationClient, fedClient *gomatrixserverlib.FederationClient,
homeservers map[string]struct{}, homeservers map[gomatrixserverlib.ServerName]struct{},
) (profiles []authtypes.Profile) { ) (profiles []authtypes.Profile) {
jsonBody, err := json.Marshal(req) jsonBody, err := json.Marshal(req)
if err != nil { if err != nil {
@ -96,7 +114,7 @@ func bulkFetchUserDirectoriesFromServers(
// concurrently query for public rooms // concurrently query for public rooms
reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5) reqctx, reqcancel := context.WithTimeout(ctx, time.Second*5)
for hs := range homeservers { for hs := range homeservers {
go func(homeserverDomain string) { go func(homeserverDomain gomatrixserverlib.ServerName) {
defer wg.Done() defer wg.Done()
util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for users") util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for users")
@ -115,7 +133,7 @@ func bulkFetchUserDirectoriesFromServers(
return return
} }
for _, profile := range res.Profiles { for _, profile := range res.Profiles {
profile.ServerName = homeserverDomain profile.ServerName = string(homeserverDomain)
// atomically send a room or stop // atomically send a room or stop
select { select {
case profileCh <- profile: case profileCh <- profile:

2
go.mod
View File

@ -39,7 +39,7 @@ require (
github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0
github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/mattn/go-sqlite3 v1.14.10 github.com/mattn/go-sqlite3 v1.14.10
github.com/morikuni/aec v1.0.0 // indirect github.com/morikuni/aec v1.0.0 // indirect

4
go.sum
View File

@ -943,8 +943,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 h1:IINbE/0jSYGb7M31StazufyIQdYWSivRlhuns3JYPOM= github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0 h1:IINbE/0jSYGb7M31StazufyIQdYWSivRlhuns3JYPOM=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= github.com/matrix-org/gomatrixserverlib v0.0.0-20220317164600-0980b7f341e0/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo=
github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278 h1:lRrvMMv7x1FIVW1mcBdU89lvbgAXKz6RyYR0VQTAr3E= github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8 h1:nMRg2106UjC98dAW236yJp9gavS4247fRftY/avkNZY=
github.com/matrix-org/pinecone v0.0.0-20220323142759-6fb077377278/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/pinecone v0.0.0-20220330132624-fb51a311e4b8/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=