feat: Add OpenTelemetry metrics instrumentation with Prometheus scrape endpoint

* setup open telemetry metrics for transactions and balances

* instrument metrics on transactions, balance, queue and worker

* setup test for verifying metrics export

* setup deployment config for prometheus

* add auth config for metrics

* enable backward compatibility for OTLP endpoint for tracing

* add documentation for supported metrics

* fix double call on transaction queue name for tracking queue_enqueued_total

* setup metrics using init function and update contributing.md to use act for testing ci

---------

Co-authored-by: jerry enebeli <jerryenebeli@gmail.com>

Author: lawale

CONTRIBUTING.md

@@ -87,6 +87,22 @@ When your change affects schema or persistence behavior:
 - Follow the existing migration naming pattern.
 - Include tests that validate the new behavior.
 
+## Running CI Locally with `act`
+
+You can run the GitHub Actions CI pipeline locally using [`act`](https://github.com/nektos/act) before pushing:
+
+1. Install `act`:
+
+```bash
+brew install act    # macOS
+```
+
+2. Make sure Docker is running, then execute the test workflow:
+
+```bash
+act -W .github/workflows/go.yml
+```
+
 ## Pull Request Checklist
 
 Before opening a PR, ensure:

api/api.go

@@ -160,7 +160,13 @@ func NewAPI(b *blnk.Blnk) *Api {
 	auth := middleware.NewAuthMiddleware(b)
 	r.Use(middleware.RateLimitMiddleware(conf))
 	r.Use(middleware.SecurityHeaders())
-	r.Use(otelgin.Middleware("BLNK"))
+	r.Use(otelgin.Middleware("BLNK",
+		otelgin.WithFilter(func(r *http.Request) bool {
+			// Exclude high-frequency operational endpoints from tracing
+			// to avoid polluting the trace feed with noise.
+			return r.URL.Path != "/metrics"
+		}),
+	))
 
 	r.GET("/", func(c *gin.Context) {
 		c.JSON(200, "server running...")

api/middleware/auth.go

@@ -192,6 +192,12 @@ func (m *AuthMiddleware) Authenticate() gin.HandlerFunc {
 			return
 		}
 
+		// Skip X-Blnk-Key auth for metrics endpoint, it uses its own bearer token auth
+		if c.Request != nil && c.Request.URL != nil && c.Request.URL.Path == "/metrics" {
+			c.Next()
+			return
+		}
+
 		// Check if secure mode is enabled
 		conf, err := config.Fetch()
 		if err == nil && conf != nil && !conf.Server.Secure {

api/middleware/middleware.go

@@ -17,6 +17,8 @@ limitations under the License.
 package middleware
 
 import (
+	"crypto/subtle"
+	"net/http"
 	"strings"
 	"time"
 
@@ -64,6 +66,93 @@ func RateLimitMiddleware(conf *config.Configuration) gin.HandlerFunc {
 	}
 }
 
+// MetricsAuth returns a middleware that controls access to the /metrics endpoint.
+//
+// Behavior based on secure mode and token configuration:
+//   - Secure mode OFF, no token: open access (no auth required)
+//   - Secure mode OFF, token set: require bearer token
+//   - Secure mode ON, token set: require bearer token
+//   - Secure mode ON, no token: block all access (misconfiguration)
+//
+// When authentication is required, requests must include "Authorization: Bearer <token>".
+// This uses the standard Authorization header that Prometheus natively supports via
+// its scrape_configs authorization block.
+func MetricsAuth(secure bool, token string) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if secure && token == "" {
+			c.AbortWithStatusJSON(http.StatusForbidden, gin.H{
+				"error": "Metrics endpoint unavailable: metrics_bearer_token must be configured when secure mode is enabled",
+			})
+			return
+		}
+
+		if token == "" {
+			c.Next()
+			return
+		}
+
+		auth := c.GetHeader("Authorization")
+		if auth == "" {
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Authorization required for metrics endpoint"})
+			return
+		}
+
+		const prefix = "Bearer "
+		if !strings.HasPrefix(auth, prefix) {
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Authorization header must use Bearer scheme"})
+			return
+		}
+
+		provided := auth[len(prefix):]
+		if subtle.ConstantTimeCompare([]byte(provided), []byte(token)) != 1 {
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Invalid bearer token"})
+			return
+		}
+
+		c.Next()
+	}
+}
+
+// MetricsAuthHandler wraps an http.Handler with bearer token authentication.
+// This is the non-Gin equivalent of MetricsAuth, used for the worker monitoring server
+// which uses a standard http.ServeMux instead of Gin.
+// Same secure mode logic as MetricsAuth: blocks access when secure=true and token is empty.
+func MetricsAuthHandler(secure bool, token string, next http.Handler) http.Handler {
+	if secure && token == "" {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusForbidden)
+			_, _ = w.Write([]byte(`{"error":"Metrics endpoint unavailable: metrics_bearer_token must be configured when secure mode is enabled"}`))
+		})
+	}
+
+	if token == "" {
+		return next
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		auth := r.Header.Get("Authorization")
+		if auth == "" {
+			http.Error(w, `{"error":"Authorization required for metrics endpoint"}`, http.StatusUnauthorized)
+			return
+		}
+
+		const prefix = "Bearer "
+		if !strings.HasPrefix(auth, prefix) {
+			http.Error(w, `{"error":"Authorization header must use Bearer scheme"}`, http.StatusUnauthorized)
+			return
+		}
+
+		provided := auth[len(prefix):]
+		if subtle.ConstantTimeCompare([]byte(provided), []byte(token)) != 1 {
+			http.Error(w, `{"error":"Invalid bearer token"}`, http.StatusUnauthorized)
+			return
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}
+
 // SecurityHeaders sets security headers to the response.
 // It sets the following headers:
 // - X-Content-Type-Options: nosniff

balance.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/blnkfinance/blnk/config"
 	"github.com/blnkfinance/blnk/internal/filter"
+	"github.com/blnkfinance/blnk/internal/metrics"
 	"github.com/blnkfinance/blnk/internal/notification"
 	"github.com/blnkfinance/blnk/model"
 	"github.com/sirupsen/logrus"
@@ -236,6 +237,7 @@ func (l *Blnk) CreateBalance(ctx context.Context, balance model.Balance) (model.
 		return model.Balance{}, err
 	}
 	l.postBalanceActions(ctx, &balance)
+	metrics.BalanceCreatedTotal.Add(ctx, 1)
 	span.AddEvent("Balance created", trace.WithAttributes(attribute.String("balance.id", balance.BalanceID)))
 	return balance, nil
 }

cmd/server.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/blnkfinance/blnk"
 	"github.com/blnkfinance/blnk/api"
+	"github.com/blnkfinance/blnk/api/middleware"
 	"github.com/blnkfinance/blnk/config"
 	"github.com/blnkfinance/blnk/database"
 	"github.com/blnkfinance/blnk/internal/search"
@@ -180,7 +181,17 @@ func healthCheckHandler(c *gin.Context) {
 
 func initializeRouter(b *blnkInstance) *gin.Engine {
 	router := api.NewAPI(b.blnk).Router()
-	router.GET("/health", healthCheckHandler) // Add health check route
+	router.GET("/health", healthCheckHandler)
+	if h := trace.MetricsHandler(); h != nil {
+		cfg, _ := config.Fetch()
+		var secure bool
+		var token string
+		if cfg != nil {
+			secure = cfg.Server.Secure
+			token = cfg.Server.MetricsBearerToken
+		}
+		router.GET("/metrics", middleware.MetricsAuth(secure, token), gin.WrapH(h))
+	}
 	return router
 }
 
@@ -300,16 +311,14 @@ func serverCommands(b *blnkInstance) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			ctx := context.Background()
 
-			// Initialize router
-			router := initializeRouter(b)
-
 			// Load configuration
 			cfg, err := config.Fetch()
 			if err != nil {
 				logrus.Error(err)
 			}
 
-			// Initialize telemetry and observability
+			// Initialize telemetry and observability before the router,
+			// so MetricsHandler() is available when routes are registered.
 			phClient, shutdown, err := initializeTelemetryAndObservability(ctx, cfg)
 			if err != nil {
 				logrus.Fatal(err)
@@ -325,6 +334,9 @@ func serverCommands(b *blnkInstance) *cobra.Command {
 				defer phClient.Close()
 			}
 
+			// Initialize router (after OTel so /metrics handler is available)
+			router := initializeRouter(b)
+
 			// Initialize TypeSense
 			tsClient, err := initializeTypeSense(ctx, cfg)
 			if err != nil {

cmd/workers.go

@@ -31,12 +31,17 @@ import (
 	"go.opentelemetry.io/otel"
 
 	"github.com/blnkfinance/blnk"
+	"github.com/blnkfinance/blnk/api/middleware"
 	"github.com/blnkfinance/blnk/config"
 	"github.com/blnkfinance/blnk/internal/hotpairs"
+	"github.com/blnkfinance/blnk/internal/metrics"
 	"github.com/blnkfinance/blnk/internal/notification"
 	redis_db "github.com/blnkfinance/blnk/internal/redis-db"
 	"github.com/blnkfinance/blnk/internal/search"
+	trace "github.com/blnkfinance/blnk/internal/traces"
 	"github.com/blnkfinance/blnk/model"
+	"go.opentelemetry.io/otel/attribute"
+	otelmetric "go.opentelemetry.io/otel/metric"
 
 	"github.com/hibiken/asynq"
 	"github.com/hibiken/asynqmon"
@@ -56,6 +61,8 @@ func (b *blnkInstance) processTransaction(ctx context.Context, t *asynq.Task) er
 	ctx, span := otel.Tracer("blnk.transactions.worker").Start(ctx, "Process Transaction From Redis Queue")
 	defer span.End()
 
+	startTime := time.Now()
+
 	var txn model.Transaction
 	if err := json.Unmarshal(t.Payload(), &txn); err != nil {
 		logrus.Error(err)
@@ -69,6 +76,19 @@ func (b *blnkInstance) processTransaction(ctx context.Context, t *asynq.Task) er
 		return nil
 	}
 
+	handled, err := b.blnk.TryRecordQueuedTransactionBatch(ctx, &txn)
+	if b.cnf.Queue.EnableHotLane && t.Type() == b.cnf.Queue.HotQueueName {
+		handled, err = b.blnk.TryRecordQueuedTransactionBatchForHotLane(ctx, &txn)
+	}
+	if err != nil {
+		logrus.WithError(err).Warnf("coalesced processing attempt failed for transaction %s", txn.TransactionID)
+	}
+	if handled {
+		metrics.QueueProcessingDuration.Record(ctx, time.Since(startTime).Seconds(),
+			otelmetric.WithAttributes(attribute.String("result", "success")),
+		)
+		return nil
+	}
 	_, err = b.blnk.ProcessQueuedTransaction(ctx, &txn, b.cnf.Queue.EnableHotLane && t.Type() == b.cnf.Queue.HotQueueName)
 	if err != nil {
 		// Handle reference already used error
@@ -94,6 +114,9 @@ func (b *blnkInstance) processTransaction(ctx context.Context, t *asynq.Task) er
 
 			logrus.Infof("Insufficient funds for transaction %s, retry attempt %d/%d",
 				txn.TransactionID, retryCount, b.cnf.Queue.MaxRetryAttempts)
+			metrics.WorkerRetriesTotal.Add(ctx, 1,
+				otelmetric.WithAttributes(attribute.String("reason", "insufficient_funds")),
+			)
 			return err // This will trigger a retry
 		}
 
@@ -121,9 +144,16 @@ func (b *blnkInstance) processTransaction(ctx context.Context, t *asynq.Task) er
 		}
 
 		logrus.Infof("Transaction %s pushed back for retry due to error: %v", txn.TransactionID, err)
+		metrics.WorkerRetriesTotal.Add(ctx, 1,
+			otelmetric.WithAttributes(attribute.String("reason", "other")),
+		)
 		return err
 	}
 
+	logrus.Infof("Transaction %s processed successfully", txn.TransactionID)
+	metrics.QueueProcessingDuration.Record(ctx, time.Since(startTime).Seconds(),
+		otelmetric.WithAttributes(attribute.String("result", "success")),
+	)
 	return nil
 }
 
@@ -527,6 +557,9 @@ func startMonitoringServer(conf *config.Configuration) *http.Server {
 	})
 
 	monitoringMux.Handle("/monitoring/", asynqmonHandler)
+	if h := trace.MetricsHandler(); h != nil {
+		monitoringMux.Handle("/metrics", middleware.MetricsAuthHandler(conf.Server.Secure, conf.Server.MetricsBearerToken, h))
+	}
 
 	monitoringAddr := fmt.Sprintf(":%s", conf.Queue.MonitoringPort)
 	srv := &http.Server{

config/config.go

@@ -90,13 +90,14 @@ var (
 var ConfigStore atomic.Value
 
 type ServerConfig struct {
-	SSL             bool   `json:"ssl" envconfig:"BLNK_SERVER_SSL"`
-	CertStoragePath string `json:"cert_storage_path" envconfig:"BLNK_CERT_STORAGE_PATH"`
-	Secure          bool   `json:"secure" envconfig:"BLNK_SERVER_SECURE"`
-	SecretKey       string `json:"secret_key" envconfig:"BLNK_SERVER_SECRET_KEY"`
-	Domain          string `json:"domain" envconfig:"BLNK_SERVER_SSL_DOMAIN"`
-	Email           string `json:"ssl_email" envconfig:"BLNK_SERVER_SSL_EMAIL"`
-	Port            string `json:"port" envconfig:"BLNK_SERVER_PORT"`
+	SSL                bool   `json:"ssl" envconfig:"BLNK_SERVER_SSL"`
+	CertStoragePath    string `json:"cert_storage_path" envconfig:"BLNK_CERT_STORAGE_PATH"`
+	Secure             bool   `json:"secure" envconfig:"BLNK_SERVER_SECURE"`
+	SecretKey          string `json:"secret_key" envconfig:"BLNK_SERVER_SECRET_KEY"`
+	Domain             string `json:"domain" envconfig:"BLNK_SERVER_SSL_DOMAIN"`
+	Email              string `json:"ssl_email" envconfig:"BLNK_SERVER_SSL_EMAIL"`
+	Port               string `json:"port" envconfig:"BLNK_SERVER_PORT"`
+	MetricsBearerToken string `json:"metrics_bearer_token" envconfig:"BLNK_METRICS_BEARER_TOKEN"`
 }
 
 type DataSourceConfig struct {

docker-compose.dev.yaml

@@ -20,7 +20,7 @@ services:
     restart: on-failure
     environment:
       TZ: ${TZ:-Etc/UTC}
-      OTEL_EXPORTER_OTLP_ENDPOINT: jaeger:4318
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://jaeger:4318/v1/traces
     ports:
       - "5001:5001"
       - "80:80"
@@ -38,7 +38,9 @@ services:
     restart: on-failure
     entrypoint: [ "blnk", "workers"]
     environment:
-      OTEL_EXPORTER_OTLP_ENDPOINT: jaeger:4318
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://jaeger:4318/v1/traces
+    ports:
+      - "5004:5004"
     depends_on:
       - redis
       - postgres
@@ -107,6 +109,19 @@ services:
       timeout: 5s
       retries: 3
 
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    profiles:
+      - monitoring
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    depends_on:
+      - server
+      - worker
+
 volumes:
   pg_data:
   typesense_data: