Add metrics for internal API requests (#2310)

* Add response size and requests total to internal handler

* Move MustRegister calls to New* funcs

* Move MustRegister back to init

* Init at some place, minimize changes
This commit is contained in:
Till 2022-04-08 12:24:40 +02:00 committed by GitHub
parent 25d99c4473
commit e8dd37d533
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 37 additions and 32 deletions

View File

@ -64,11 +64,6 @@ const (
sessionIDLength = 24 sessionIDLength = 24
) )
func init() {
// Register prometheus metrics. They must be registered to be exposed.
prometheus.MustRegister(amtRegUsers)
}
// sessionsDict keeps track of completed auth stages for each session. // sessionsDict keeps track of completed auth stages for each session.
// It shouldn't be passed by value because it contains a mutex. // It shouldn't be passed by value because it contains a mutex.
type sessionsDict struct { type sessionsDict struct {

View File

@ -37,6 +37,7 @@ import (
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util" "github.com/matrix-org/util"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -60,6 +61,8 @@ func Setup(
extRoomsProvider api.ExtraPublicRoomsProvider, extRoomsProvider api.ExtraPublicRoomsProvider,
mscCfg *config.MSCs, natsClient *nats.Conn, mscCfg *config.MSCs, natsClient *nats.Conn,
) { ) {
prometheus.MustRegister(amtRegUsers, sendEventDuration)
rateLimits := httputil.NewRateLimits(&cfg.RateLimiting) rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg) userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg)

View File

@ -46,10 +46,6 @@ var (
userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents
) )
func init() {
prometheus.MustRegister(sendEventDuration)
}
var sendEventDuration = prometheus.NewHistogramVec( var sendEventDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: "dendrite", Namespace: "dendrite",

View File

@ -29,6 +29,7 @@ import (
userapi "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util" "github.com/matrix-org/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -53,6 +54,10 @@ func Setup(
servers federationAPI.ServersInRoomProvider, servers federationAPI.ServersInRoomProvider,
producer *producers.SyncAPIProducer, producer *producers.SyncAPIProducer,
) { ) {
prometheus.MustRegister(
pduCountTotal, eduCountTotal,
)
v2keysmux := keyMux.PathPrefix("/v2").Subrouter() v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
v1fedmux := fedMux.PathPrefix("/v1").Subrouter() v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
v2fedmux := fedMux.PathPrefix("/v2").Subrouter() v2fedmux := fedMux.PathPrefix("/v2").Subrouter()

View File

@ -74,12 +74,6 @@ var (
) )
) )
func init() {
prometheus.MustRegister(
pduCountTotal, eduCountTotal,
)
}
var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse
// Send implements /_matrix/federation/v1/send/{txnID} // Send implements /_matrix/federation/v1/send/{txnID}

View File

@ -169,8 +169,9 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request)
return promhttp.InstrumentHandlerCounter( return promhttp.InstrumentHandlerCounter(
promauto.NewCounterVec( promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: metricsName, Name: metricsName,
Help: "Total number of http requests for HTML resources", Help: "Total number of http requests for HTML resources",
Namespace: "dendrite",
}, },
[]string{"code"}, []string{"code"},
), ),
@ -201,7 +202,28 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse
h.ServeHTTP(w, req) h.ServeHTTP(w, req)
} }
return http.HandlerFunc(withSpan) return promhttp.InstrumentHandlerCounter(
promauto.NewCounterVec(
prometheus.CounterOpts{
Name: metricsName + "_requests_total",
Help: "Total number of internal API calls",
Namespace: "dendrite",
},
[]string{"code"},
),
promhttp.InstrumentHandlerResponseSize(
promauto.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "dendrite",
Name: metricsName + "_response_size_bytes",
Help: "A histogram of response sizes for requests.",
Buckets: []float64{200, 500, 900, 1500, 5000, 15000, 50000, 100000},
},
[]string{},
),
http.HandlerFunc(withSpan),
),
)
} }
// MakeFedAPI makes an http.Handler that checks matrix federation authentication. // MakeFedAPI makes an http.Handler that checks matrix federation authentication.

View File

@ -167,6 +167,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) {
// will look to see if we have a worker for that room which has its // will look to see if we have a worker for that room which has its
// own consumer. If we don't, we'll start one. // own consumer. If we don't, we'll start one.
func (r *Inputer) Start() error { func (r *Inputer) Start() error {
prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration)
_, err := r.JetStream.Subscribe( _, err := r.JetStream.Subscribe(
"", // This is blank because we specified it in BindStream. "", // This is blank because we specified it in BindStream.
func(m *nats.Msg) { func(m *nats.Msg) {
@ -421,10 +422,6 @@ func (r *Inputer) WriteOutputEvents(roomID string, updates []api.OutputEvent) er
return nil return nil
} }
func init() {
prometheus.MustRegister(roomserverInputBackpressure)
}
var roomserverInputBackpressure = prometheus.NewGaugeVec( var roomserverInputBackpressure = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Namespace: "dendrite", Namespace: "dendrite",

View File

@ -37,10 +37,6 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
func init() {
prometheus.MustRegister(processRoomEventDuration)
}
// TODO: Does this value make sense? // TODO: Does this value make sense?
const MaximumMissingProcessingTime = time.Minute * 2 const MaximumMissingProcessingTime = time.Minute * 2

View File

@ -67,6 +67,9 @@ func NewRequestPool(
streams *streams.Streams, notifier *notifier.Notifier, streams *streams.Streams, notifier *notifier.Notifier,
producer PresencePublisher, producer PresencePublisher,
) *RequestPool { ) *RequestPool {
prometheus.MustRegister(
activeSyncRequests, waitingSyncRequests,
)
rp := &RequestPool{ rp := &RequestPool{
db: db, db: db,
cfg: cfg, cfg: cfg,
@ -183,12 +186,6 @@ func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device)
rp.lastseen.Store(device.UserID+device.ID, time.Now()) rp.lastseen.Store(device.UserID+device.ID, time.Now())
} }
func init() {
prometheus.MustRegister(
activeSyncRequests, waitingSyncRequests,
)
}
var activeSyncRequests = prometheus.NewGauge( var activeSyncRequests = prometheus.NewGauge(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Namespace: "dendrite", Namespace: "dendrite",