Единый observability-стек: Prometheus + Loki + Grafana + Caddy auth-гейт
Отдельный от db+vaultwarden compose-файл (разный жизненный цикл, свой deploy-workflow) — единственное место сбора для backend, лендинга и каждой прокси-ноды, которые физически живут на других серверах. Prometheus скрейпит наружу (сам ходит в /metrics на тех хостах — там нужно только открыть порт метрик для IP этого VPS). Loki, наоборот, принимает пуши логов ИЗВНЕ — порт 3100 публикуется наружу и обязательно фильтруется firewall'ом по IP тех хостов. Grafana не торчит в интернет напрямую (127.0.0.1-only) — единственный вход через Caddy с forward_auth, гейтящим по cookie-сессии netrunner-backend (GET /api/v1/admin/observability/check, Owner/Moderator). 3 provisioned дашборда (backend/landing/прокси-ноды). Список scrape-таргетов для прокси-нод (observability/targets/nodes.json) генерится и заливается автоматически из netrunner-proxy при каждом деплое нод — вручную ничего не поддерживается. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"uid": "netrunner-backend",
|
||||
"title": "Netrunner — Backend",
|
||||
"timezone": "utc",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "HTTP requests/sec (by status)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total{job=\"netrunner-backend\"}[5m])) by (status)",
|
||||
"legendFormat": "{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "HTTP latency p95/p50",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"netrunner-backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"netrunner-backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Errors/sec (by type)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(app_errors_total{job=\"netrunner-backend\"}[5m])) by (type)",
|
||||
"legendFormat": "{{type}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Payments/sec (by provider, status)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(payments_total{job=\"netrunner-backend\"}[5m])) by (provider, status)",
|
||||
"legendFormat": "{{provider}}/{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "VPN nodes online / total",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "nodes_online{job=\"netrunner-backend\"}", "legendFormat": "online" },
|
||||
{ "expr": "nodes_total{job=\"netrunner-backend\"}", "legendFormat": "total" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Support tickets/hour (created vs closed)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(support_tickets_total{job=\"netrunner-backend\"}[1h])) by (status) * 3600",
|
||||
"legendFormat": "{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Referral signups + partner commission",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(referral_signups_total{job=\"netrunner-backend\"}[1h])) * 3600",
|
||||
"legendFormat": "referral signups/hour"
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(partner_commission_total{job=\"netrunner-backend\"}[1h])) by (currency) * 3600",
|
||||
"legendFormat": "commission/hour {{currency}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"uid": "netrunner-landing",
|
||||
"title": "Netrunner — Landing",
|
||||
"timezone": "utc",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Next.js requests/sec (by status)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total{job=\"netrunner-landing\"}[5m])) by (status)",
|
||||
"legendFormat": "{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "ws-proxy (гостевая книга/спидтест) requests/sec",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total{job=\"netrunner-landing-ws-proxy\"}[5m])) by (path, status)",
|
||||
"legendFormat": "{{path}} {{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "ws-proxy latency p95/p50",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"netrunner-landing-ws-proxy\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"netrunner-landing-ws-proxy\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Next.js process (CPU/memory)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(process_cpu_seconds_total{job=\"netrunner-landing\"}[5m])",
|
||||
"legendFormat": "CPU"
|
||||
},
|
||||
{
|
||||
"expr": "process_resident_memory_bytes{job=\"netrunner-landing\"}",
|
||||
"legendFormat": "RSS bytes"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"uid": "netrunner-proxy-nodes",
|
||||
"title": "Netrunner — Прокси-ноды (ядро)",
|
||||
"timezone": "utc",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "node",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"query": "label_values(netrunner_connections_active, node)",
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": { "text": "All", "value": "$__all" },
|
||||
"refresh": 2
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Активные соединения (по ноде)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "netrunner_connections_active{job=\"netrunner-proxy-nodes\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Новых соединений/сек (по ноде)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(netrunner_connections_total{job=\"netrunner-proxy-nodes\", node=~\"$node\"}[5m])) by (node)",
|
||||
"legendFormat": "{{node}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Auth validate latency p95 (по ноде)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(netrunner_auth_validate_duration_seconds_bucket{job=\"netrunner-proxy-nodes\", node=~\"$node\"}[5m])) by (le, node))",
|
||||
"legendFormat": "{{node}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Auth failures/sec (по ноде)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(netrunner_auth_failures_total{job=\"netrunner-proxy-nodes\", node=~\"$node\"}[5m])) by (node)",
|
||||
"legendFormat": "{{node}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Circuit breaker открыт (0/1, по ноде)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "Prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "netrunner_circuit_breaker_open{job=\"netrunner-proxy-nodes\", node=~\"$node\"}",
|
||||
"legendFormat": "{{node}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user