Add instrumentation to the backend layer

Publish per-op counters for all backends (ldap, cache).

backend/instrumented/instrumented_backend.go

+package instrumented
+
+import (
+	"context"
+
+	as "git.autistici.org/ai3/accountserver"
+	ct "git.autistici.org/ai3/go-common/ldap/compositetypes"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var counters = prometheus.NewCounterVec(prometheus.CounterOpts{
+	Name: "accountserver_backend_ops_total",
+	Help: "Total backend operations",
+}, []string{"backend", "op"})
+
+func init() {
+	prometheus.MustRegister(counters)
+}
+
+type instrumentedBackend struct {
+	as.Backend
+
+	name string
+}
+
+// Wrap the given backend with per-op instrumentation counters.
+func Wrap(b as.Backend, name string) as.Backend {
+	return &instrumentedBackend{
+		Backend: b,
+		name:    name,
+	}
+}
+
+func (b *instrumentedBackend) NewTransaction() (as.TX, error) {
+	tx, err := b.Backend.NewTransaction()
+	return &instrumentedTX{
+		TX:   tx,
+		name: b.name,
+	}, err
+}
+
+type instrumentedTX struct {
+	as.TX
+
+	name string
+}
+
+func (tx instrumentedTX) UpdateResource(ctx context.Context, r *as.Resource) error {
+	counters.WithLabelValues(tx.name, "UpdateResource").Inc()
+	return tx.TX.UpdateResource(ctx, r)
+}
+
+func (tx instrumentedTX) CreateResources(ctx context.Context, u *as.User, r []*as.Resource) ([]*as.Resource, error) {
+	counters.WithLabelValues(tx.name, "CreateResources").Inc()
+	return tx.TX.CreateResources(ctx, u, r)
+}
+
+func (tx instrumentedTX) SetResourcePassword(ctx context.Context, r *as.Resource, s string) error {
+	counters.WithLabelValues(tx.name, "SetResourcePassword").Inc()
+	return tx.TX.SetResourcePassword(ctx, r, s)
+}
+
+func (tx instrumentedTX) FindResource(ctx context.Context, r as.FindResourceRequest) (*as.RawResource, error) {
+	counters.WithLabelValues(tx.name, "FindResource").Inc()
+	return tx.TX.FindResource(ctx, r)
+}
+
+func (tx instrumentedTX) HasAnyResource(ctx context.Context, r []as.FindResourceRequest) (bool, error) {
+	counters.WithLabelValues(tx.name, "HasAnyResource").Inc()
+	return tx.TX.HasAnyResource(ctx, r)
+}
+
+func (tx instrumentedTX) GetUser(ctx context.Context, s string) (*as.RawUser, error) {
+	counters.WithLabelValues(tx.name, "GetUser").Inc()
+	return tx.TX.GetUser(ctx, s)
+}
+
+func (tx instrumentedTX) UpdateUser(ctx context.Context, u *as.User) error {
+	counters.WithLabelValues(tx.name, "UpdateUser").Inc()
+	return tx.TX.UpdateUser(ctx, u)
+}
+
+func (tx instrumentedTX) CreateUser(ctx context.Context, u *as.User) (*as.User, error) {
+	counters.WithLabelValues(tx.name, "CreateUser").Inc()
+	return tx.TX.CreateUser(ctx, u)
+}
+
+func (tx instrumentedTX) SetUserPassword(ctx context.Context, u *as.User, s string) error {
+	counters.WithLabelValues(tx.name, "SetUserPassword").Inc()
+	return tx.TX.SetUserPassword(ctx, u, s)
+}
+
+func (tx instrumentedTX) SetAccountRecoveryHint(ctx context.Context, u *as.User, s1 string, s2 string) error {
+	counters.WithLabelValues(tx.name, "SetAccountRecoveryHint").Inc()
+	return tx.TX.SetAccountRecoveryHint(ctx, u, s1, s2)
+}
+
+func (tx instrumentedTX) DeleteAccountRecoveryHint(ctx context.Context, u *as.User) error {
+	counters.WithLabelValues(tx.name, "DeleteAccountRecoveryHint").Inc()
+	return tx.TX.DeleteAccountRecoveryHint(ctx, u)
+}
+
+func (tx instrumentedTX) SetUserEncryptionKeys(ctx context.Context, u *as.User, k []*ct.EncryptedKey) error {
+	counters.WithLabelValues(tx.name, "SetUserEncryptionKeys").Inc()
+	return tx.TX.SetUserEncryptionKeys(ctx, u, k)
+}
+
+func (tx instrumentedTX) SetUserEncryptionPublicKey(ctx context.Context, u *as.User, k []byte) error {
+	counters.WithLabelValues(tx.name, "SetUserEncryptionPublicKey").Inc()
+	return tx.TX.SetUserEncryptionPublicKey(ctx, u, k)
+}
+
+func (tx instrumentedTX) SetApplicationSpecificPassword(ctx context.Context, u *as.User, a *as.AppSpecificPasswordInfo, s string) error {
+	counters.WithLabelValues(tx.name, "SetApplicationSpecificPassword").Inc()
+	return tx.TX.SetApplicationSpecificPassword(ctx, u, a, s)
+}
+
+func (tx instrumentedTX) DeleteApplicationSpecificPassword(ctx context.Context, u *as.User, s string) error {
+	counters.WithLabelValues(tx.name, "DeleteApplicationSpecificPassword").Inc()
+	return tx.TX.DeleteApplicationSpecificPassword(ctx, u, s)
+}
+
+func (tx instrumentedTX) SetUserTOTPSecret(ctx context.Context, u *as.User, s string) error {
+	counters.WithLabelValues(tx.name, "SetUserTOTPSecret").Inc()
+	return tx.TX.SetUserTOTPSecret(ctx, u, s)
+}
+
+func (tx instrumentedTX) DeleteUserTOTPSecret(ctx context.Context, u *as.User) error {
+	counters.WithLabelValues(tx.name, "DeleteUserTOTPSecret").Inc()
+	return tx.TX.DeleteUserTOTPSecret(ctx, u)
+}
+
+// Lightweight user search (backend-specific pattern).
+// Returns list of matching usernames.
+func (tx instrumentedTX) SearchUser(ctx context.Context, s string, i int) ([]string, error) {
+	counters.WithLabelValues(tx.name, "SearchUser").Inc()
+	return tx.TX.SearchUser(ctx, s, i)
+}
+
+// Resource search (backend-specific pattern).
+func (tx instrumentedTX) SearchResource(ctx context.Context, s string, i int) ([]*as.RawResource, error) {
+	counters.WithLabelValues(tx.name, "SearchResource").Inc()
+	return tx.TX.SearchResource(ctx, s, i)
+}
+
+// Resource ACL check (does not necessarily hit the database).
+func (tx instrumentedTX) CanAccessResource(ctx context.Context, s string, r *as.Resource) bool {
+	counters.WithLabelValues(tx.name, "CanAccessResource").Inc()
+	return tx.TX.CanAccessResource(ctx, s, r)
+}
+
+// Return the next (or any, really) available user ID.
+//func (tx instrumentedTX) NextUID(context.Context) (int, error) {}

cmd/accountserver/main.go

@@ -15,6 +15,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	cachebackend "git.autistici.org/ai3/accountserver/backend/cache"
+	insbackend "git.autistici.org/ai3/accountserver/backend/instrumented"
 	ldapbackend "git.autistici.org/ai3/accountserver/backend/ldap"
 	webappdbbackend "git.autistici.org/ai3/accountserver/backend/webappdb"
 	"git.autistici.org/ai3/accountserver/server"
@@ -163,7 +164,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	be, err := ldapbackend.NewLDAPBackend(
+	ldapBE, err := ldapbackend.NewLDAPBackend(
 		config.LDAP.URI,
 		config.LDAP.BindDN,
 		bindPw,
@@ -172,18 +173,23 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	be := insbackend.Wrap(ldapBE, "ldap")
 
 	// If the cache is enabled, create the cache backend wrapper. It
 	// also acts as a http.Handler for the replicated cache
 	// invalidation RPC, so we're going to have to route its endpoint
 	// on the main Server later.
-	var cache *cachebackend.CacheBackend
+	var cacheBE *cachebackend.CacheBackend
 	if config.Cache.Enabled {
-		cache, err = cachebackend.Wrap(be, config.Replication.Peers, config.Replication.TLS)
+		cacheBE, err = cachebackend.Wrap(
+			be,
+			config.Replication.Peers,
+			config.Replication.TLS,
+		)
 		if err != nil {
 			log.Fatal(err)
 		}
-		be = cache
+		be = insbackend.Wrap(cacheBE, "cache")
 	}
 
 	// Enable lookups to the webappdb (FreeWVS) service. Errors
@@ -206,8 +212,8 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	if cache != nil {
-		as.Handle(cachebackend.InvalidateURLPath, cache)
+	if cacheBE != nil {
+		as.Handle(cachebackend.InvalidateURLPath, cacheBE)
 	}
 
 	// Start the HTTP server.