feat(metrics): implement comprehensive Prometheus metrics across all endpoints

- Add MetricsMiddleware for automatic HTTP request tracking
- Track request duration, status codes, and error rates
- Integrate metrics into investigation endpoints with business metrics
- Update BaseAgent to use centralized metrics manager
- Add metrics decorators (@track_time, @count_calls) for easy integration
- Track agent task execution with duration and status
- Monitor investigation lifecycle (created, completed, failed)
- Record anomaly detection metrics with confidence scores
- Setup HTTP-specific metrics (requests, errors, slow requests)
- Enable /api/v1/observability/metrics endpoint for Prometheus scraping
- Track active investigations gauge for real-time monitoring
- Replace legacy monitoring with centralized metrics manager

src/agents/deodoro.py

@@ -17,7 +17,7 @@ from pydantic import BaseModel, Field as PydanticField
 
 from src.core import AgentStatus, get_logger
 from src.core.exceptions import AgentError, AgentExecutionError
-from src.core.monitoring import AGENT_TASK_COUNT, AGENT_TASK_DURATION
+from src.infrastructure.observability.metrics import metrics_manager, BusinessMetrics
 import time
 
 
@@ -188,23 +188,29 @@ class BaseAgent(ABC):
                     retry=retries,
                 )
                 
-                # Process the message with timing
-                with AGENT_TASK_DURATION.labels(
-                    agent_type=self.name,
-                    task_type=action
-                ).time():
-                    response = await self.process(message, context)
+                # Process the message
+                response = await self.process(message, context)
                 
                 # Calculate processing time
                 processing_time = (datetime.utcnow() - start_time).total_seconds() * 1000
                 response.processing_time_ms = processing_time
                 
-                # Record successful execution
-                AGENT_TASK_COUNT.labels(
-                    agent_type=self.name,
+                # Record metrics using centralized metrics manager
+                metrics_manager.increment_counter(
+                    "cidadao_ai_agent_tasks_total",
+                    labels={
+                        "agent_name": self.name,
+                        "task_type": action,
+                        "status": "completed"
+                    }
+                )
+                
+                BusinessMetrics.record_agent_task(
+                    agent_name=self.name,
                     task_type=action,
-                    status="completed"
-                ).inc()
+                    duration_seconds=processing_time / 1000.0,
+                    status="success"
+                )
                 
                 # Update status
                 self.status = AgentStatus.COMPLETED
@@ -233,11 +239,14 @@ class BaseAgent(ABC):
                 )
                 
                 # Record retry attempt
-                AGENT_TASK_COUNT.labels(
-                    agent_type=self.name,
-                    task_type=action,
-                    status="retry"
-                ).inc()
+                metrics_manager.increment_counter(
+                    "cidadao_ai_agent_tasks_total",
+                    labels={
+                        "agent_name": self.name,
+                        "task_type": action,
+                        "status": "retry"
+                    }
+                )
                 
                 retries += 1
                 if retries <= self.max_retries:
@@ -245,11 +254,21 @@ class BaseAgent(ABC):
                     await self._wait(2 ** retries)
         
         # All retries exhausted - record failure
-        AGENT_TASK_COUNT.labels(
-            agent_type=self.name,
+        metrics_manager.increment_counter(
+            "cidadao_ai_agent_tasks_total",
+            labels={
+                "agent_name": self.name,
+                "task_type": action,
+                "status": "failed"
+            }
+        )
+        
+        BusinessMetrics.record_agent_task(
+            agent_name=self.name,
             task_type=action,
+            duration_seconds=(datetime.utcnow() - start_time).total_seconds(),
             status="failed"
-        ).inc()
+        )
         
         self.status = AgentStatus.ERROR
         

src/api/app.py

@@ -26,6 +26,7 @@ from src.api.middleware.authentication import AuthenticationMiddleware
 from src.api.middleware.logging_middleware import LoggingMiddleware
 from src.api.middleware.security import SecurityMiddleware
 from src.api.middleware.compression import CompressionMiddleware
+from src.api.middleware.metrics_middleware import MetricsMiddleware, setup_http_metrics
 from src.infrastructure.observability import (
     CorrelationMiddleware,
     tracing_manager,
@@ -63,6 +64,9 @@ async def lifespan(app: FastAPI):
         build_info={"deployment": "hf-fastapi"}
     )
     
+    # Setup HTTP metrics
+    setup_http_metrics()
+    
     # Initialize global resources here
     # - Database connections
     # - Background tasks
@@ -168,6 +172,9 @@ app.add_middleware(
 # Add observability middleware
 app.add_middleware(CorrelationMiddleware, generate_request_id=True)
 
+# Add metrics middleware for automatic HTTP metrics
+app.add_middleware(MetricsMiddleware)
+
 # Add compression middleware
 from src.api.middleware.compression import add_compression_middleware
 add_compression_middleware(

src/api/middleware/metrics_middleware.py

@@ -0,0 +1,238 @@
+"""
+Prometheus metrics middleware for automatic HTTP request tracking.
+
+This middleware automatically records metrics for all HTTP requests,
+including duration, status codes, and error rates.
+"""
+
+import time
+from typing import Callable
+
+from fastapi import Request, Response
+from starlette.middleware.base import BaseHTTPMiddleware
+
+from src.core import get_logger
+from src.infrastructure.observability.metrics import metrics_manager, BusinessMetrics
+
+logger = get_logger(__name__)
+
+
+class MetricsMiddleware(BaseHTTPMiddleware):
+    """Middleware for automatic Prometheus metrics collection."""
+    
+    def __init__(self, app):
+        """Initialize metrics middleware."""
+        super().__init__(app)
+        self.logger = get_logger(__name__)
+    
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        """Process request with automatic metrics collection."""
+        start_time = time.time()
+        
+        # Skip metrics endpoint to avoid recursion
+        if request.url.path == "/api/v1/observability/metrics":
+            return await call_next(request)
+        
+        # Extract path template (FastAPI route) for grouping
+        path_template = self._get_path_template(request)
+        method = request.method
+        
+        try:
+            # Process request
+            response = await call_next(request)
+            
+            # Calculate duration
+            duration = time.time() - start_time
+            
+            # Record metrics
+            self._record_request_metrics(
+                method=method,
+                path=path_template,
+                status_code=response.status_code,
+                duration=duration
+            )
+            
+            return response
+            
+        except Exception as exc:
+            # Calculate duration even for errors
+            duration = time.time() - start_time
+            
+            # Record error metrics
+            self._record_request_metrics(
+                method=method,
+                path=path_template,
+                status_code=500,  # Default error status
+                duration=duration,
+                error=True
+            )
+            
+            # Re-raise the exception
+            raise exc
+    
+    def _get_path_template(self, request: Request) -> str:
+        """
+        Get the path template from FastAPI route.
+        
+        This extracts the route pattern (e.g., /users/{user_id})
+        instead of the actual path (e.g., /users/123) to avoid
+        high cardinality in metrics.
+        """
+        # Try to get route from request scope
+        if hasattr(request, "scope") and "route" in request.scope:
+            route = request.scope["route"]
+            if hasattr(route, "path"):
+                return route.path
+        
+        # Fallback to actual path, but try to generalize it
+        path = request.url.path
+        
+        # Common patterns to generalize
+        # Replace UUIDs
+        import re
+        path = re.sub(
+            r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
+            '{uuid}',
+            path,
+            flags=re.IGNORECASE
+        )
+        
+        # Replace numeric IDs
+        path = re.sub(r'/\d+', '/{id}', path)
+        
+        # Limit cardinality for unknown paths
+        if path.count('/') > 5:
+            path = '/unknown/deep/path'
+        
+        return path
+    
+    def _record_request_metrics(
+        self,
+        method: str,
+        path: str,
+        status_code: int,
+        duration: float,
+        error: bool = False
+    ):
+        """Record HTTP request metrics."""
+        # HTTP request duration histogram
+        metrics_manager.observe_histogram(
+            "cidadao_ai_request_duration_seconds",
+            duration,
+            labels={
+                "method": method.upper(),
+                "endpoint": path,
+                "status_code": str(status_code)
+            }
+        )
+        
+        # HTTP request counter
+        metrics_manager.increment_counter(
+            "cidadao_ai_http_requests_total",
+            labels={
+                "method": method.upper(),
+                "endpoint": path,
+                "status_code": str(status_code),
+                "status": "error" if error or status_code >= 400 else "success"
+            }
+        )
+        
+        # Error rate tracking
+        if error or status_code >= 400:
+            metrics_manager.increment_counter(
+                "cidadao_ai_http_errors_total",
+                labels={
+                    "method": method.upper(),
+                    "endpoint": path,
+                    "status_code": str(status_code),
+                    "error_type": self._get_error_type(status_code)
+                }
+            )
+        
+        # Track slow requests
+        if duration > 5.0:  # Requests taking more than 5 seconds
+            metrics_manager.increment_counter(
+                "cidadao_ai_slow_requests_total",
+                labels={
+                    "method": method.upper(),
+                    "endpoint": path,
+                    "duration_bucket": self._get_duration_bucket(duration)
+                }
+            )
+        
+        # Update concurrent requests gauge (simplified - in production use proper tracking)
+        active_requests = getattr(self, '_active_requests', 0)
+        metrics_manager.set_gauge(
+            "cidadao_ai_http_requests_in_progress",
+            active_requests,
+            labels={"method": method.upper()}
+        )
+    
+    def _get_error_type(self, status_code: int) -> str:
+        """Categorize error types based on status code."""
+        if 400 <= status_code < 500:
+            return "client_error"
+        elif 500 <= status_code < 600:
+            return "server_error"
+        else:
+            return "unknown_error"
+    
+    def _get_duration_bucket(self, duration: float) -> str:
+        """Categorize request duration into buckets."""
+        if duration < 1:
+            return "0-1s"
+        elif duration < 5:
+            return "1-5s"
+        elif duration < 10:
+            return "5-10s"
+        elif duration < 30:
+            return "10-30s"
+        else:
+            return "30s+"
+
+
+def setup_http_metrics():
+    """Setup HTTP-specific metrics if not already registered."""
+    # HTTP requests total counter
+    try:
+        from src.infrastructure.observability.metrics import MetricConfig, MetricType
+        
+        metrics_manager.register_metric(
+            MetricConfig(
+                name="cidadao_ai_http_requests_total",
+                description="Total HTTP requests received",
+                labels=["method", "endpoint", "status_code", "status"]
+            ),
+            MetricType.COUNTER
+        )
+        
+        metrics_manager.register_metric(
+            MetricConfig(
+                name="cidadao_ai_http_errors_total",
+                description="Total HTTP errors",
+                labels=["method", "endpoint", "status_code", "error_type"]
+            ),
+            MetricType.COUNTER
+        )
+        
+        metrics_manager.register_metric(
+            MetricConfig(
+                name="cidadao_ai_slow_requests_total",
+                description="Total slow HTTP requests",
+                labels=["method", "endpoint", "duration_bucket"]
+            ),
+            MetricType.COUNTER
+        )
+        
+        metrics_manager.register_metric(
+            MetricConfig(
+                name="cidadao_ai_http_requests_in_progress",
+                description="HTTP requests currently being processed",
+                labels=["method"]
+            ),
+            MetricType.GAUGE
+        )
+        
+        logger.info("HTTP metrics initialized")
+    except Exception as e:
+        logger.warning(f"Some HTTP metrics may already be registered: {e}")

src/api/routes/investigations.py

@@ -19,6 +19,7 @@ from src.core import get_logger
 from src.agents import InvestigatorAgent, AgentContext
 from src.api.middleware.authentication import get_current_user
 from src.tools import TransparencyAPIFilter
+from src.infrastructure.observability.metrics import track_time, count_calls, BusinessMetrics
 
 
 logger = get_logger(__name__)
@@ -105,6 +106,8 @@ _active_investigations: Dict[str, Dict[str, Any]] = {}
 
 
 @router.post("/start", response_model=Dict[str, str])
+@count_calls("cidadao_ai_investigation_requests_total", labels={"operation": "start"})
+@track_time("cidadao_ai_investigation_start_duration_seconds")
 async def start_investigation(
     request: InvestigationRequest,
     background_tasks: BackgroundTasks,
@@ -150,6 +153,13 @@ async def start_investigation(
         user_id=current_user.get("user_id"),
     )
     
+    # Track business metrics
+    BusinessMetrics.record_investigation_created(
+        priority="medium",
+        user_type="authenticated"
+    )
+    BusinessMetrics.update_active_investigations(len(_active_investigations))
+    
     return {
         "investigation_id": investigation_id,
         "status": "started",
@@ -394,6 +404,7 @@ async def _run_investigation(investigation_id: str, request: InvestigationReques
     This function runs the actual anomaly detection using InvestigatorAgent.
     """
     investigation = _active_investigations[investigation_id]
+    start_time = datetime.utcnow()
     
     try:
         # Update status
@@ -462,6 +473,9 @@ async def _run_investigation(investigation_id: str, request: InvestigationReques
         investigation["progress"] = 1.0
         investigation["current_phase"] = "completed"
         
+        # Calculate duration
+        duration = (datetime.utcnow() - start_time).total_seconds()
+        
         logger.info(
             "investigation_completed",
             investigation_id=investigation_id,
@@ -469,6 +483,23 @@ async def _run_investigation(investigation_id: str, request: InvestigationReques
             records_analyzed=investigation["records_processed"],
         )
         
+        # Track business metrics
+        BusinessMetrics.record_investigation_completed(
+            investigation_type=request.data_source,
+            duration_seconds=duration,
+            priority="medium"
+        )
+        BusinessMetrics.update_active_investigations(len(_active_investigations) - 1)
+        
+        # Track anomalies found
+        for result in results:
+            BusinessMetrics.record_anomaly_detected(
+                anomaly_type=result.anomaly_type,
+                severity=result.severity,
+                data_source=request.data_source,
+                confidence_score=result.confidence
+            )
+        
     except Exception as e:
         logger.error(
             "investigation_failed",