From 25dcf801806bbca4ac76060f595591881b67de32 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Wed, 24 Nov 2021 13:55:44 +0100 Subject: [PATCH] Ratelimit requests to /media/r0/download|upload (#2020) * Add /media/r0/config handler Signed-off-by: Till Faelligen * Add rate limiting to media api * Rename variable * Add passing tests * Don't send multiple headers Co-authored-by: Neil Alexander --- clientapi/routing/routing.go | 46 +++++++++---------- .../personalities/mediaapi.go | 2 +- .../httputil}/rate_limiting.go | 12 ++--- mediaapi/mediaapi.go | 6 ++- mediaapi/routing/routing.go | 42 +++++++++++++++-- setup/monolith.go | 2 +- sytest-whitelist | 1 + 7 files changed, 74 insertions(+), 37 deletions(-) rename {clientapi/routing => internal/httputil}/rate_limiting.go (92%) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 813d9d16..9263c66b 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -61,7 +61,7 @@ func Setup( extRoomsProvider api.ExtraPublicRoomsProvider, mscCfg *config.MSCs, ) { - rateLimits := newRateLimits(&cfg.RateLimiting) + rateLimits := httputil.NewRateLimits(&cfg.RateLimiting) userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg) unstableFeatures := map[string]bool{ @@ -127,7 +127,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -143,7 +143,7 @@ func Setup( if mscCfg.Enabled("msc2753") { r0mux.Handle("/peek/{roomIDOrAlias}", httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -163,7 +163,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/join", httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -177,7 +177,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/leave", httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -211,7 +211,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/invite", httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -329,14 +329,14 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Register(req, userAPI, accountDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return RegisterAvailable(req, cfg, accountDB) @@ -410,7 +410,7 @@ func Setup( r0mux.Handle("/rooms/{roomID}/typing/{userID}", httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -466,7 +466,7 @@ func Setup( r0mux.Handle("/account/whoami", httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Whoami(req, device) @@ -475,7 +475,7 @@ func Setup( r0mux.Handle("/account/password", httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Password(req, userAPI, accountDB, device, cfg) @@ -484,7 +484,7 @@ func Setup( r0mux.Handle("/account/deactivate", httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Deactivate(req, userInteractiveAuth, userAPI, device) @@ -495,7 +495,7 @@ func Setup( r0mux.Handle("/login", httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Login(req, accountDB, userAPI, cfg) @@ -552,7 +552,7 @@ func Setup( r0mux.Handle("/profile/{userID}/avatar_url", httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -577,7 +577,7 @@ func Setup( r0mux.Handle("/profile/{userID}/displayname", httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -617,7 +617,7 @@ func Setup( // Element logs get flooded unless this is handled r0mux.Handle("/presence/{userID}/status", httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } // TODO: Set presence (probably the responsibility of a presence server not clientapi) @@ -630,7 +630,7 @@ func Setup( r0mux.Handle("/voip/turnServer", httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return RequestTurnServer(req, device, cfg) @@ -709,7 +709,7 @@ func Setup( r0mux.Handle("/user/{userID}/openid/request_token", httputil.MakeAuthAPI("openid_request_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -722,7 +722,7 @@ func Setup( r0mux.Handle("/user_directory/search", httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } postContent := struct { @@ -767,7 +767,7 @@ func Setup( r0mux.Handle("/rooms/{roomID}/read_markers", httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -780,7 +780,7 @@ func Setup( r0mux.Handle("/rooms/{roomID}/forget", httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -884,7 +884,7 @@ func Setup( r0mux.Handle("/capabilities", httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return GetCapabilities(req, rsAPI) @@ -1100,7 +1100,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}", httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) diff --git a/cmd/dendrite-polylith-multi/personalities/mediaapi.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go index 00a2d56a..fa9d36a3 100644 --- a/cmd/dendrite-polylith-multi/personalities/mediaapi.go +++ b/cmd/dendrite-polylith-multi/personalities/mediaapi.go @@ -24,7 +24,7 @@ func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() client := base.CreateClient() - mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client) + mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, &base.Cfg.ClientAPI.RateLimiting, userAPI, client) base.SetupAndServeHTTP( base.Cfg.MediaAPI.InternalAPI.Listen, diff --git a/clientapi/routing/rate_limiting.go b/internal/httputil/rate_limiting.go similarity index 92% rename from clientapi/routing/rate_limiting.go rename to internal/httputil/rate_limiting.go index 5291caba..c4f47c7b 100644 --- a/clientapi/routing/rate_limiting.go +++ b/internal/httputil/rate_limiting.go @@ -1,4 +1,4 @@ -package routing +package httputil import ( "net/http" @@ -10,7 +10,7 @@ import ( "github.com/matrix-org/util" ) -type rateLimits struct { +type RateLimits struct { limits map[string]chan struct{} limitsMutex sync.RWMutex cleanMutex sync.RWMutex @@ -19,8 +19,8 @@ type rateLimits struct { cooloffDuration time.Duration } -func newRateLimits(cfg *config.RateLimiting) *rateLimits { - l := &rateLimits{ +func NewRateLimits(cfg *config.RateLimiting) *RateLimits { + l := &RateLimits{ limits: make(map[string]chan struct{}), enabled: cfg.Enabled, requestThreshold: cfg.Threshold, @@ -32,7 +32,7 @@ func newRateLimits(cfg *config.RateLimiting) *rateLimits { return l } -func (l *rateLimits) clean() { +func (l *RateLimits) clean() { for { // On a 30 second interval, we'll take an exclusive write // lock of the entire map and see if any of the channels are @@ -52,7 +52,7 @@ func (l *rateLimits) clean() { } } -func (l *rateLimits) rateLimit(req *http.Request) *util.JSONResponse { +func (l *RateLimits) Limit(req *http.Request) *util.JSONResponse { // If rate limiting is disabled then do nothing. if !l.enabled { return nil diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 811d8e4a..c010981c 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -26,7 +26,9 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. func AddPublicRoutes( - router *mux.Router, cfg *config.MediaAPI, + router *mux.Router, + cfg *config.MediaAPI, + rateLimit *config.RateLimiting, userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, ) { @@ -36,6 +38,6 @@ func AddPublicRoutes( } routing.Setup( - router, cfg, mediaDB, userAPI, client, + router, cfg, rateLimit, mediaDB, userAPI, client, ) } diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 917a8596..44f9a9d6 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -15,16 +15,16 @@ package routing import ( + "encoding/json" "net/http" "strings" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/dendrite/setup/config" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -32,6 +32,12 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) +// configResponse is the response to GET /_matrix/media/r0/config +// https://matrix.org/docs/spec/client_server/latest#get-matrix-media-r0-config +type configResponse struct { + UploadSize config.FileSizeBytes `json:"m.upload.size"` +} + // Setup registers the media API HTTP handlers // // Due to Setup being used to call many other functions, a gocyclo nolint is @@ -40,10 +46,13 @@ import ( func Setup( publicAPIMux *mux.Router, cfg *config.MediaAPI, + rateLimit *config.RateLimiting, db storage.Database, userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, ) { + rateLimits := httputil.NewRateLimits(rateLimit) + r0mux := publicAPIMux.PathPrefix("/r0").Subrouter() v1mux := publicAPIMux.PathPrefix("/v1").Subrouter() @@ -54,31 +63,46 @@ func Setup( uploadHandler := httputil.MakeAuthAPI( "upload", userAPI, func(req *http.Request, dev *userapi.Device) util.JSONResponse { + if r := rateLimits.Limit(req); r != nil { + return *r + } return Upload(req, cfg, dev, db, activeThumbnailGeneration) }, ) + configHandler := httputil.MakeAuthAPI("config", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + if r := rateLimits.Limit(req); r != nil { + return *r + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: configResponse{UploadSize: *cfg.MaxFileSizeBytes}, + } + }) + r0mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/config", configHandler).Methods(http.MethodGet, http.MethodOptions) v1mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions) activeRemoteRequests := &types.ActiveRemoteRequests{ MXCToResult: map[string]*types.RemoteRequestResult{}, } - downloadHandler := makeDownloadAPI("download", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration) + downloadHandler := makeDownloadAPI("download", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration) r0mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed r0mux.Handle("/thumbnail/{serverName}/{mediaId}", - makeDownloadAPI("thumbnail", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration), + makeDownloadAPI("thumbnail", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration), ).Methods(http.MethodGet, http.MethodOptions) } func makeDownloadAPI( name string, cfg *config.MediaAPI, + rateLimits *httputil.RateLimits, db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, @@ -99,6 +123,16 @@ func makeDownloadAPI( // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") + // Ratelimit requests + if r := rateLimits.Limit(req); r != nil { + if err := json.NewEncoder(w).Encode(r); err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusTooManyRequests) + return + } + vars, _ := httputil.URLDecodeMapValues(mux.Vars(req)) serverName := gomatrixserverlib.ServerName(vars["serverName"]) diff --git a/setup/monolith.go b/setup/monolith.go index b076e990..e6c95522 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -68,7 +68,7 @@ func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ss m.KeyRing, m.RoomserverAPI, m.FederationAPI, m.EDUInternalAPI, m.KeyAPI, &m.Config.MSCs, nil, ) - mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, m.UserAPI, m.Client) + mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting, m.UserAPI, m.Client) syncapi.AddPublicRoutes( process, csMux, m.UserAPI, m.RoomserverAPI, m.KeyAPI, m.FedClient, &m.Config.SyncAPI, diff --git a/sytest-whitelist b/sytest-whitelist index d074d42b..558eb29a 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -556,6 +556,7 @@ can fetch self-signing keys over federation Changing master key notifies local users Changing user-signing key notifies local users Inbound federation correctly handles soft failed events as extremities +Can read configuration endpoint User can create and send/receive messages in a room with version 7 local user can join room with version 7 User can invite local user to room with version 7