{ "annotations": { "list": [] }, "editable": true, "graphTooltip": 1, "id": null, "links": [], "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Суммарная нагрузка всех deal-процессов (HTTP + gRPC) в секунду. Источник — OTel-инструментация ASP.NET Core (метрика http.server.request.duration, счётчик _count), scrape /metrics :9464.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 }, "id": 1, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(http_server_request_duration_seconds_count[$__rate_interval]))", "legendFormat": "RPS", "refId": "A" } ], "title": "RPS (HTTP+gRPC) сейчас", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Активные непросроченные сессии пользователей тенантов и операторов (public.sessions + public.operator_sessions). Агрегат по всем тенантам — в метке нет tenantId (низкая кардинальность).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "deal_sessions_active", "legendFormat": "сессии", "refId": "A" } ], "title": "Активные сессии", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Суммарная глубина очереди пайплайна (new+filtered) по всем тенантам; считает фоновый DealMetricsCollector через PipelineProcessingService.QueueCountsAsync.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "deal_pipeline_queue_depth", "legendFormat": "очередь", "refId": "A" } ], "title": "Очередь пайплайна", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Суммарная глубина очереди обучения ML (MlOutbox) по всем тенантам; считает фоновый DealMetricsCollector через IMlLearningStore.CountOutboxAsync. Рост при недоступности ml-service.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "deal_ml_outbox_depth", "legendFormat": "outbox", "refId": "A" } ], "title": "Очередь ML-outbox", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "RPS по сервисам (метка service из таргетов Prometheus). Включает HTTP-запросы core и gRPC-вызовы всех 4 процессов (gRPC идёт как ASP.NET Core-запрос с route = имя метода).", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, "id": 5, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (service) (rate(http_server_request_duration_seconds_count[$__rate_interval]))", "legendFormat": "{{service}}", "refId": "A" } ], "title": "RPS по сервисам", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "p95 длительности запроса по сервисам (OTel-гистограмма http.server.request.duration). Рост при недоступности БД/внешних вызовов (ИИ-провайдер, ml/telegram-service).", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, "id": 6, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, service) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))", "legendFormat": "{{service}}", "refId": "A" } ], "title": "Длительность запроса p95 по сервисам", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Доля HTTP/gRPC-запросов с кодом ответа 5xx в секунду по сервисам (метка http_response_status_code).", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, "id": 7, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (service) (rate(http_server_request_duration_seconds_count{http_response_status_code=~\"5..\"}[$__rate_interval]))", "legendFormat": "{{service}}", "refId": "A" } ], "title": "Ошибки 5xx по сервисам", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Токены платного ИИ (по видам prompt/completion) и оценка токенов локальных ML-вызовов, токенов/сек. Инкремент — общая с ядром точка записи token_usage_events (TokenUsageRecorder).", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, "id": 8, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (type) (rate(deal_ai_tokens_total[$__rate_interval]))", "legendFormat": "ai/{{type}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(deal_ml_tokens_total[$__rate_interval]))", "legendFormat": "ml/prompt", "refId": "B" } ], "title": "Токены AI/ML, tokens/s", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Вызовы платного ИИ и локального ML в секунду (соответствуют событиям token_usage_events: kind=ai|ml).", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "id": 9, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(deal_ai_calls_total[$__rate_interval]))", "legendFormat": "AI-вызовы", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(deal_ml_calls_total[$__rate_interval]))", "legendFormat": "ML-вызовы", "refId": "B" } ], "title": "Вызовы AI/ML, calls/s", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Топ-10 типов событий аудита по частоте (метки event/actor низкокардинальные; tenantId/userId в метрики не попадают). Полная лента с актором — public.audit_log через GET /api/operator/audit.", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "id": 10, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum by (event) (rate(deal_audit_events_total[$__rate_interval])))", "legendFormat": "{{event}}", "refId": "A" } ], "title": "События аудита по типам (top-10)", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "deal", "metrics" ], "templating": { "list": [] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Deal-Metrics-Overview", "uid": "deal-metrics", "version": 1, "weekStart": "" }