Implement /thirdparty endpoints (#2831)

Implements the following endpoints 
```
GET /_matrix/client/v3/thirdparty/protocols
GET /_matrix/client/v3/thirdparty/protocols/{protocol}
GET /_matrix/client/v3/thirdparty/location
GET /_matrix/client/v3/thirdparty/location/{protocol}
GET /_matrix/client/v3/thirdparty/user
GET /_matrix/client/v3/thirdparty/user/{protocol}
```

appservice/api/query.go

@@ -19,11 +19,13 @@ package api
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AppServiceInternalAPI is used to query user and room alias data from application
@@ -41,6 +43,10 @@ type AppServiceInternalAPI interface {
 		req *UserIDExistsRequest,
 		resp *UserIDExistsResponse,
 	) error
+
+	Locations(ctx context.Context, req *LocationRequest, resp *LocationResponse) error
+	User(ctx context.Context, request *UserRequest, response *UserResponse) error
+	Protocols(ctx context.Context, req *ProtocolRequest, resp *ProtocolResponse) error
 }
 
 // RoomAliasExistsRequest is a request to an application service
@@ -77,6 +83,73 @@ type UserIDExistsResponse struct {
 	UserIDExists bool `json:"exists"`
 }
 
+const (
+	ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
+	ASUserPath     = "/_matrix/app/unstable/thirdparty/user"
+	ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
+)
+
+type ProtocolRequest struct {
+	Protocol string `json:"protocol,omitempty"`
+}
+
+type ProtocolResponse struct {
+	Protocols map[string]ASProtocolResponse `json:"protocols"`
+	Exists    bool                          `json:"exists"`
+}
+
+type ASProtocolResponse struct {
+	FieldTypes     map[string]FieldType `json:"field_types,omitempty"` // NOTSPEC: field_types is required by the spec
+	Icon           string               `json:"icon"`
+	Instances      []ProtocolInstance   `json:"instances"`
+	LocationFields []string             `json:"location_fields"`
+	UserFields     []string             `json:"user_fields"`
+}
+
+type FieldType struct {
+	Placeholder string `json:"placeholder"`
+	Regexp      string `json:"regexp"`
+}
+
+type ProtocolInstance struct {
+	Description string          `json:"desc"`
+	Icon        string          `json:"icon,omitempty"`
+	NetworkID   string          `json:"network_id,omitempty"` // NOTSPEC: network_id is required by the spec
+	Fields      json.RawMessage `json:"fields,omitempty"`     // NOTSPEC: fields is required by the spec
+}
+
+type UserRequest struct {
+	Protocol string `json:"protocol"`
+	Params   string `json:"params"`
+}
+
+type UserResponse struct {
+	Users  []ASUserResponse `json:"users,omitempty"`
+	Exists bool             `json:"exists,omitempty"`
+}
+
+type ASUserResponse struct {
+	Protocol string          `json:"protocol"`
+	UserID   string          `json:"userid"`
+	Fields   json.RawMessage `json:"fields"`
+}
+
+type LocationRequest struct {
+	Protocol string `json:"protocol"`
+	Params   string `json:"params"`
+}
+
+type LocationResponse struct {
+	Locations []ASLocationResponse `json:"locations,omitempty"`
+	Exists    bool                 `json:"exists,omitempty"`
+}
+
+type ASLocationResponse struct {
+	Alias    string          `json:"alias"`
+	Protocol string          `json:"protocol"`
+	Fields   json.RawMessage `json:"fields"`
+}
+
 // RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
 // TODO: Remove this, it's called from federationapi and clientapi but is a pure function

appservice/appservice.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -60,6 +61,8 @@ func NewInternalAPI(
 	appserviceQueryAPI := &query.AppServiceQueryAPI{
 		HTTPClient:    client,
 		Cfg:           &base.Cfg.AppServiceAPI,
+		ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{},
+		CacheMu:       sync.Mutex{},
 	}
 
 	if len(base.Cfg.Derived.ApplicationServices) == 0 {

appservice/inthttp/client.go

@@ -13,6 +13,9 @@ import (
 const (
 	AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
 	AppServiceUserIDExistsPath    = "/appservice/UserIDExists"
+	AppServiceLocationsPath       = "/appservice/locations"
+	AppServiceUserPath            = "/appservice/users"
+	AppServiceProtocolsPath       = "/appservice/protocols"
 )
 
 // httpAppServiceQueryAPI contains the URL to an appservice query API and a
@@ -58,3 +61,24 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
 		h.httpClient, ctx, request, response,
 	)
 }
+
+func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASLocation", h.appserviceURL+AppServiceLocationsPath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
+func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASUser", h.appserviceURL+AppServiceUserPath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
+func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASProtocols", h.appserviceURL+AppServiceProtocolsPath,
+		h.httpClient, ctx, request, response,
+	)
+}

appservice/inthttp/server.go

@@ -2,6 +2,7 @@ package inthttp
 
 import (
 	"github.com/gorilla/mux"
+
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 )
@@ -17,4 +18,19 @@ func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
 		AppServiceUserIDExistsPath,
 		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
 	)
+
+	internalAPIMux.Handle(
+		AppServiceProtocolsPath,
+		httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
+	)
+
+	internalAPIMux.Handle(
+		AppServiceLocationsPath,
+		httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
+	)
+
+	internalAPIMux.Handle(
+		AppServiceUserPath,
+		httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
+	)
 }

appservice/query/query.go

@@ -18,13 +18,18 @@ package query
 
 import (
 	"context"
+	"encoding/json"
+	"io"
 	"net/http"
 	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/opentracing/opentracing-go"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	opentracing "github.com/opentracing/opentracing-go"
-	log "github.com/sirupsen/logrus"
 )
 
 const roomAliasExistsPath = "/rooms/"
@@ -34,6 +39,8 @@ const userIDExistsPath = "/users/"
 type AppServiceQueryAPI struct {
 	HTTPClient    *http.Client
 	Cfg           *config.AppServiceAPI
+	ProtocolCache map[string]api.ASProtocolResponse
+	CacheMu       sync.Mutex
 }
 
 // RoomAliasExists performs a request to '/room/{roomAlias}' on all known
@@ -165,3 +172,178 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	response.UserIDExists = false
 	return nil
 }
+
+type thirdpartyResponses interface {
+	api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
+}
+
+func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
+	origURL := url
+	// try v1 and unstable appservice endpoints
+	for _, version := range []string{"v1", "unstable"} {
+		var resp *http.Response
+		var body []byte
+		asURL := strings.Replace(origURL, "unstable", version, 1)
+		resp, err = client.Get(asURL)
+		if err != nil {
+			continue
+		}
+		defer resp.Body.Close() // nolint: errcheck
+		body, err = io.ReadAll(resp.Body)
+		if err != nil {
+			continue
+		}
+		return json.Unmarshal(body, &response)
+	}
+	return err
+}
+
+func (a *AppServiceQueryAPI) Locations(
+	ctx context.Context,
+	req *api.LocationRequest,
+	resp *api.LocationResponse,
+) error {
+	params, err := url.ParseQuery(req.Params)
+	if err != nil {
+		return err
+	}
+
+	for _, as := range a.Cfg.Derived.ApplicationServices {
+		var asLocations []api.ASLocationResponse
+		params.Set("access_token", as.HSToken)
+
+		url := as.URL + api.ASLocationPath
+		if req.Protocol != "" {
+			url += "/" + req.Protocol
+		}
+
+		if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
+			log.WithError(err).Error("unable to get 'locations' from application service")
+			continue
+		}
+
+		resp.Locations = append(resp.Locations, asLocations...)
+	}
+
+	if len(resp.Locations) == 0 {
+		resp.Exists = false
+		return nil
+	}
+	resp.Exists = true
+	return nil
+}
+
+func (a *AppServiceQueryAPI) User(
+	ctx context.Context,
+	req *api.UserRequest,
+	resp *api.UserResponse,
+) error {
+	params, err := url.ParseQuery(req.Params)
+	if err != nil {
+		return err
+	}
+
+	for _, as := range a.Cfg.Derived.ApplicationServices {
+		var asUsers []api.ASUserResponse
+		params.Set("access_token", as.HSToken)
+
+		url := as.URL + api.ASUserPath
+		if req.Protocol != "" {
+			url += "/" + req.Protocol
+		}
+
+		if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
+			log.WithError(err).Error("unable to get 'user' from application service")
+			continue
+		}
+
+		resp.Users = append(resp.Users, asUsers...)
+	}
+
+	if len(resp.Users) == 0 {
+		resp.Exists = false
+		return nil
+	}
+	resp.Exists = true
+	return nil
+}
+
+func (a *AppServiceQueryAPI) Protocols(
+	ctx context.Context,
+	req *api.ProtocolRequest,
+	resp *api.ProtocolResponse,
+) error {
+
+	// get a single protocol response
+	if req.Protocol != "" {
+
+		a.CacheMu.Lock()
+		defer a.CacheMu.Unlock()
+		if proto, ok := a.ProtocolCache[req.Protocol]; ok {
+			resp.Exists = true
+			resp.Protocols = map[string]api.ASProtocolResponse{
+				req.Protocol: proto,
+			}
+			return nil
+		}
+
+		response := api.ASProtocolResponse{}
+		for _, as := range a.Cfg.Derived.ApplicationServices {
+			var proto api.ASProtocolResponse
+			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil {
+				log.WithError(err).Error("unable to get 'protocol' from application service")
+				continue
+			}
+
+			if len(response.Instances) != 0 {
+				response.Instances = append(response.Instances, proto.Instances...)
+			} else {
+				response = proto
+			}
+		}
+
+		if len(response.Instances) == 0 {
+			resp.Exists = false
+			return nil
+		}
+
+		resp.Exists = true
+		resp.Protocols = map[string]api.ASProtocolResponse{
+			req.Protocol: response,
+		}
+		a.ProtocolCache[req.Protocol] = response
+		return nil
+	}
+
+	response := make(map[string]api.ASProtocolResponse, len(a.Cfg.Derived.ApplicationServices))
+
+	for _, as := range a.Cfg.Derived.ApplicationServices {
+		for _, p := range as.Protocols {
+			var proto api.ASProtocolResponse
+			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil {
+				log.WithError(err).Error("unable to get 'protocol' from application service")
+				continue
+			}
+			existing, ok := response[p]
+			if !ok {
+				response[p] = proto
+				continue
+			}
+			existing.Instances = append(existing.Instances, proto.Instances...)
+			response[p] = existing
+		}
+	}
+
+	if len(response) == 0 {
+		resp.Exists = false
+		return nil
+	}
+
+	a.CacheMu.Lock()
+	defer a.CacheMu.Unlock()
+	a.ProtocolCache = response
+
+	resp.Exists = true
+	resp.Protocols = response
+	return nil
+}

clientapi/routing/routing.go

@@ -869,12 +869,50 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	v3mux.Handle("/thirdparty/protocols",
-		httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
-			// TODO: Return the third party protcols
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: struct{}{},
+		httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return Protocols(req, asAPI, device, "")
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/thirdparty/protocol/{protocolID}",
+		httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
 			}
+			return Protocols(req, asAPI, device, vars["protocolID"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/thirdparty/user/{protocolID}",
+		httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return User(req, asAPI, device, vars["protocolID"], req.URL.Query())
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/thirdparty/user",
+		httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return User(req, asAPI, device, "", req.URL.Query())
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/thirdparty/location/{protocolID}",
+		httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return Location(req, asAPI, device, vars["protocolID"], req.URL.Query())
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/thirdparty/location",
+		httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return Location(req, asAPI, device, "", req.URL.Query())
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 

clientapi/routing/thirdparty.go

@@ -0,0 +1,106 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+	"net/url"
+
+	"github.com/matrix-org/util"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/userapi/api"
+)
+
+// Protocols implements
+//
+//	GET /_matrix/client/v3/thirdparty/protocols/{protocol}
+//	GET /_matrix/client/v3/thirdparty/protocols
+func Protocols(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string) util.JSONResponse {
+	resp := &appserviceAPI.ProtocolResponse{}
+
+	if err := asAPI.Protocols(req.Context(), &appserviceAPI.ProtocolRequest{Protocol: protocol}, resp); err != nil {
+		return jsonerror.InternalServerError()
+	}
+	if !resp.Exists {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The protocol is unknown."),
+		}
+	}
+	if protocol != "" {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: resp.Protocols[protocol],
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp.Protocols,
+	}
+}
+
+// User implements
+//
+//	GET /_matrix/client/v3/thirdparty/user
+//	GET /_matrix/client/v3/thirdparty/user/{protocol}
+func User(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string, params url.Values) util.JSONResponse {
+	resp := &appserviceAPI.UserResponse{}
+
+	params.Del("access_token")
+	if err := asAPI.User(req.Context(), &appserviceAPI.UserRequest{
+		Protocol: protocol,
+		Params:   params.Encode(),
+	}, resp); err != nil {
+		return jsonerror.InternalServerError()
+	}
+	if !resp.Exists {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The Matrix User ID was not found"),
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp.Users,
+	}
+}
+
+// Location implements
+//
+//	GET /_matrix/client/v3/thirdparty/location
+//	GET /_matrix/client/v3/thirdparty/location/{protocol}
+func Location(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string, params url.Values) util.JSONResponse {
+	resp := &appserviceAPI.LocationResponse{}
+
+	params.Del("access_token")
+	if err := asAPI.Locations(req.Context(), &appserviceAPI.LocationRequest{
+		Protocol: protocol,
+		Params:   params.Encode(),
+	}, resp); err != nil {
+		return jsonerror.InternalServerError()
+	}
+	if !resp.Exists {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("No portal rooms were found."),
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp.Locations,
+	}
+}