Метрики /metrics для центрального Prometheus + генерация scrape-таргетов
--metrics-port (0.0.0.0, в отличие от --health-port — тот 127.0.0.1-only) отдаёт netrunner_connections_active/total, netrunner_auth_validate_duration_seconds, netrunner_auth_failures_total, netrunner_circuit_breaker_open. Тот же hand-rolled HTTP-паттерн, что и health.rs — без HTTP-фреймворка ради одного эндпоинта. scripts/generate-prometheus-targets.mjs генерит список scrape-таргетов из nodes.json — job update-observability-targets в deploy.yml гоняет его при каждом деплое нод и заливает на VPS с данными (netrunner-data/observability/targets/nodes.json), чтобы центральный Prometheus не терял из виду ноды без ручной правки конфига. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -23,6 +23,15 @@
|
||||
#
|
||||
# Repo Variable (vars.*, не secret — публичный URL, не даёт доступа сам по себе):
|
||||
# - BACKEND_URL — публичный адрес control plane (WEB_APP_BASE_URL у бэкенда)
|
||||
#
|
||||
# Job update-observability-targets (см. ниже) обновляет список scrape-таргетов
|
||||
# для центрального Prometheus (netrunner-data/docker-compose.observability.yml,
|
||||
# job "netrunner-proxy-nodes") при КАЖДОМ прогоне (независимо от input target,
|
||||
# т.к. использует полный nodes.json) — ему нужны СВОИ копии секретов/vars
|
||||
# VPS с данными, продублировать в Settings этого репозитория (те же значения,
|
||||
# что уже есть в netrunner-data):
|
||||
# - vars.DATA_VPS_IP
|
||||
# - secrets.DATA_SSH_PRIVATE_KEY
|
||||
name: Deploy Proxy Nodes
|
||||
|
||||
on:
|
||||
@@ -99,7 +108,37 @@ jobs:
|
||||
--decoy-host "${{ matrix.decoy_host }}" \
|
||||
--require-auth \
|
||||
--backend-url "${{ vars.BACKEND_URL }}" \
|
||||
--health-port 9091
|
||||
--health-port 9091 \
|
||||
--metrics-port 9093
|
||||
|
||||
docker image prune -f
|
||||
echo "✅ ${{ matrix.name }} обновлена."
|
||||
|
||||
# Независимо от того, какая конкретно нода деплоилась (target может быть
|
||||
# одной нодой, не "all") — список scrape-таргетов всегда генерится из ПОЛНОГО
|
||||
# nodes.json, чтобы центральный Prometheus не терял из виду остальные ноды.
|
||||
update-observability-targets:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Generate targets/nodes.json from nodes.json
|
||||
run: node scripts/generate-prometheus-targets.mjs nodes.json > nodes.targets.json
|
||||
|
||||
- name: Upload to observability VPS
|
||||
uses: appleboy/scp-action@v0.1.7
|
||||
with:
|
||||
host: ${{ vars.DATA_VPS_IP }}
|
||||
username: root
|
||||
key: ${{ secrets.DATA_SSH_PRIVATE_KEY }}
|
||||
source: "nodes.targets.json"
|
||||
target: "/root/netrunner-data/observability/targets"
|
||||
strip_components: 0
|
||||
|
||||
- name: Rename to nodes.json (file_sd_configs ждёт именно это имя)
|
||||
uses: appleboy/ssh-action@v1
|
||||
with:
|
||||
host: ${{ vars.DATA_VPS_IP }}
|
||||
username: root
|
||||
key: ${{ secrets.DATA_SSH_PRIVATE_KEY }}
|
||||
script: mv /root/netrunner-data/observability/targets/nodes.targets.json /root/netrunner-data/observability/targets/nodes.json
|
||||
|
||||
Generated
+185
@@ -572,6 +572,15 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.21"
|
||||
@@ -771,6 +780,17 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "evmap"
|
||||
version = "11.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8"
|
||||
dependencies = [
|
||||
"hashbag",
|
||||
"left-right",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.4.1"
|
||||
@@ -929,6 +949,21 @@ dependencies = [
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generator"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"log",
|
||||
"rustversion",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
@@ -1025,6 +1060,12 @@ dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbag"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
@@ -1459,6 +1500,17 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
||||
|
||||
[[package]]
|
||||
name = "left-right"
|
||||
version = "0.11.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f0c21e4c8ff95f487fb34e6f9182875f42c84cef966d29216bf115d9bba835a"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
"loom",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.184"
|
||||
@@ -1512,6 +1564,19 @@ version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "loom"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"generator",
|
||||
"scoped-tls",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.16.3"
|
||||
@@ -1548,6 +1613,48 @@ version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "metrics"
|
||||
version = "0.24.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
|
||||
dependencies = [
|
||||
"portable-atomic",
|
||||
"rapidhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "metrics-exporter-prometheus"
|
||||
version = "0.18.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"evmap",
|
||||
"indexmap",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"quanta",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "metrics-util"
|
||||
version = "0.20.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"hashbrown 0.16.1",
|
||||
"metrics",
|
||||
"quanta",
|
||||
"rand 0.9.2",
|
||||
"rand_xoshiro",
|
||||
"rapidhash",
|
||||
"sketches-ddsketch",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
@@ -1654,6 +1761,8 @@ dependencies = [
|
||||
"async-trait",
|
||||
"clap",
|
||||
"dashmap",
|
||||
"metrics",
|
||||
"metrics-exporter-prometheus",
|
||||
"netrunner-core",
|
||||
"netrunner-logger",
|
||||
"reqwest",
|
||||
@@ -1869,6 +1978,21 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quanta"
|
||||
version = "0.12.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"raw-cpuid",
|
||||
"wasi",
|
||||
"web-sys",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
@@ -2001,6 +2125,33 @@ version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba"
|
||||
|
||||
[[package]]
|
||||
name = "rand_xoshiro"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
|
||||
dependencies = [
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rapidhash"
|
||||
version = "4.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "raw-cpuid"
|
||||
version = "11.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.18"
|
||||
@@ -2220,6 +2371,12 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scoped-tls"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
@@ -2373,6 +2530,12 @@ version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
|
||||
|
||||
[[package]]
|
||||
name = "sketches-ddsketch"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
@@ -3274,6 +3437,22 @@ dependencies = [
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
@@ -3283,6 +3462,12 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env node
|
||||
// Генерит targets-файл для Prometheus file_sd_configs (см.
|
||||
// netrunner-data/observability/prometheus.yml, job "netrunner-proxy-nodes")
|
||||
// из nodes.json этого репозитория — список нод меняется независимо от
|
||||
// прод-конфига Prometheus, руками синхронизировать его на каждый деплой
|
||||
// ноды неудобно и хрупко.
|
||||
//
|
||||
// Порт метрик у всех нод одинаковый (9093 — см. server/netrunner-server.service,
|
||||
// templates/init_node.sh в netrunner-backend, .gitea/workflows/deploy.yml
|
||||
// здесь же), меняется только host — поэтому в самом nodes.json отдельного
|
||||
// поля под него нет.
|
||||
//
|
||||
// Использование: node scripts/generate-prometheus-targets.mjs nodes.json > targets.json
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
const METRICS_PORT = 9093;
|
||||
|
||||
const [, , nodesPath] = process.argv;
|
||||
if (!nodesPath) {
|
||||
console.error("Использование: generate-prometheus-targets.mjs <nodes.json>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const nodes = JSON.parse(readFileSync(nodesPath, "utf-8"));
|
||||
|
||||
const targets = nodes
|
||||
.filter((n) => !n.host?.startsWith("CHANGE_ME"))
|
||||
.map((n) => ({
|
||||
targets: [`${n.host}:${METRICS_PORT}`],
|
||||
labels: { node: n.name },
|
||||
}));
|
||||
|
||||
process.stdout.write(JSON.stringify(targets, null, 2) + "\n");
|
||||
@@ -15,3 +15,5 @@ serde = { version = "1", features = ["derive"] }
|
||||
reqwest = { version = "0.13.2", features = ["json"] }
|
||||
async-trait = "0.1.89"
|
||||
dashmap = "6.1.0"
|
||||
metrics = "0.24.3"
|
||||
metrics-exporter-prometheus = { version = "0.18.1", default-features = false }
|
||||
|
||||
@@ -5,7 +5,7 @@ After=network.target
|
||||
[Service]
|
||||
User=root
|
||||
WorkingDirectory=/root/netr-core
|
||||
ExecStart=/root/netr-core/netrunner-server --host=0.0.0.0 --port=8443 --health-port=9092
|
||||
ExecStart=/root/netr-core/netrunner-server --host=0.0.0.0 --port=8443 --health-port=9092 --metrics-port=9094
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=65535
|
||||
|
||||
@@ -5,7 +5,7 @@ After=network.target
|
||||
[Service]
|
||||
User=root
|
||||
WorkingDirectory=/root/netr-core
|
||||
ExecStart=/root/netr-core/netrunner-server --host=0.0.0.0 --port=443 --health-port=9091
|
||||
ExecStart=/root/netr-core/netrunner-server --host=0.0.0.0 --port=443 --health-port=9091 --metrics-port=9093
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
LimitNOFILE=65535
|
||||
|
||||
@@ -62,6 +62,7 @@ impl Circuit {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.consecutive_failures = 0;
|
||||
state.open_until = None;
|
||||
metrics::gauge!("netrunner_circuit_breaker_open").set(0.0);
|
||||
}
|
||||
|
||||
/// Считать неудачей только сетевые ошибки/таймауты/5xx — HTTP 401/403 на
|
||||
@@ -71,6 +72,7 @@ impl Circuit {
|
||||
state.consecutive_failures += 1;
|
||||
if state.consecutive_failures >= FAILURE_THRESHOLD {
|
||||
state.open_until = Some(Instant::now() + CIRCUIT_OPEN_COOLDOWN);
|
||||
metrics::gauge!("netrunner_circuit_breaker_open").set(1.0);
|
||||
warn!(
|
||||
failures = state.consecutive_failures,
|
||||
"Circuit breaker разомкнут: бэкенд не отвечает {} раз подряд",
|
||||
@@ -140,6 +142,7 @@ struct UsageResponse {
|
||||
impl AuthValidator for BackendClient {
|
||||
async fn validate(&self, token: &str) -> Result<UserQuota, AppError> {
|
||||
if token.is_empty() {
|
||||
metrics::counter!("netrunner_auth_failures_total").increment(1);
|
||||
return Err(AppError::new(
|
||||
netrunner_logger::ERR_AUTH_FAILED,
|
||||
"Доступ запрещен",
|
||||
@@ -158,6 +161,7 @@ impl AuthValidator for BackendClient {
|
||||
return Err(Self::circuit_open_error());
|
||||
}
|
||||
|
||||
let request_start = Instant::now();
|
||||
let resp = self
|
||||
.http
|
||||
.post(format!("{}/api/v1/internal/validate", self.base_url))
|
||||
@@ -173,6 +177,8 @@ impl AuthValidator for BackendClient {
|
||||
e.to_string(),
|
||||
)
|
||||
})?;
|
||||
metrics::histogram!("netrunner_auth_validate_duration_seconds")
|
||||
.record(request_start.elapsed().as_secs_f64());
|
||||
|
||||
// 5xx — признак нездоровья самого бэкенда, считается неудачей для
|
||||
// circuit breaker'а. 4xx (например 401 на невалидный токен) — это
|
||||
@@ -181,6 +187,7 @@ impl AuthValidator for BackendClient {
|
||||
self.circuit.record_failure();
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
metrics::counter!("netrunner_auth_failures_total").increment(1);
|
||||
return Err(AppError::new(
|
||||
netrunner_logger::ERR_AUTH_FAILED,
|
||||
"Доступ запрещен",
|
||||
|
||||
@@ -10,8 +10,8 @@
|
||||
//! [`ServerDiagnosticsLogger::snapshot_all_sessions`]).
|
||||
|
||||
use netrunner_core::net::diagnostics::{
|
||||
self, DiagnosticsEvent, DiagnosticsSnapshot, DiagnosticsStore, TunnelMetrics,
|
||||
current_timestamp_ms,
|
||||
self, current_timestamp_ms, DiagnosticsEvent, DiagnosticsSnapshot, DiagnosticsStore,
|
||||
TunnelMetrics,
|
||||
};
|
||||
use netrunner_core::net::SessionManager;
|
||||
use netrunner_logger::{error, info, warn};
|
||||
@@ -192,10 +192,7 @@ impl ServerDiagnosticsLogger {
|
||||
let mut rotated = self.log_path.clone();
|
||||
rotated.set_extension("jsonl.1");
|
||||
tokio::fs::rename(&self.log_path, &rotated).await?;
|
||||
info!(
|
||||
"Diagnostics log rotated → {}",
|
||||
rotated.display()
|
||||
);
|
||||
info!("Diagnostics log rotated → {}", rotated.display());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
+26
-1
@@ -13,6 +13,7 @@
|
||||
mod backend_client;
|
||||
mod diagnostics;
|
||||
mod health;
|
||||
mod metrics_server;
|
||||
mod network;
|
||||
use clap::Parser;
|
||||
use netrunner_core::net::AuthValidator;
|
||||
@@ -88,6 +89,13 @@ struct Args {
|
||||
/// этой же машине). Не задан — health-эндпоинт выключен.
|
||||
#[arg(long)]
|
||||
health_port: Option<u16>,
|
||||
|
||||
/// Порт для `/metrics` (Prometheus text exposition) — в отличие от
|
||||
/// `--health-port`, биндится на 0.0.0.0 (нужен для скрейпа удалённым
|
||||
/// центральным Prometheus), ОБЯЗАТЕЛЬНО зафайрволить на IP
|
||||
/// observability-VPS. Не задан — метрики выключены.
|
||||
#[arg(long)]
|
||||
metrics_port: Option<u16>,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
@@ -106,8 +114,15 @@ fn main() {
|
||||
None
|
||||
};
|
||||
|
||||
// Регистрируется один раз, до первого metrics::counter!/gauge!/histogram! —
|
||||
// если --metrics-port не задан, вызовы макросов молча уходят в
|
||||
// no-op recorder по умолчанию (штатное поведение крейта metrics).
|
||||
let metrics_handle = args
|
||||
.metrics_port
|
||||
.map(|_| metrics_server::install_recorder());
|
||||
|
||||
let net = Network::new(
|
||||
args.host,
|
||||
args.host.clone(),
|
||||
args.port,
|
||||
args.decoy_host,
|
||||
auth,
|
||||
@@ -128,6 +143,16 @@ fn main() {
|
||||
net.run(run_token).await;
|
||||
});
|
||||
|
||||
if let (Some(port), Some(handle)) = (args.metrics_port, metrics_handle) {
|
||||
let metrics_token = token.clone();
|
||||
tokio::spawn(metrics_server::run(
|
||||
"0.0.0.0".to_string(),
|
||||
port,
|
||||
handle,
|
||||
metrics_token,
|
||||
));
|
||||
}
|
||||
|
||||
shutdown_signal(token).await;
|
||||
if let Err(e) = run_handle.await {
|
||||
error!(error = ?e, "Задача сервера завершилась с паникой при остановке");
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
//! `/metrics` для центрального Prometheus (см.
|
||||
//! `netrunner-data/docker-compose.observability.yml`) — в отличие от
|
||||
//! `/health` (127.0.0.1-only, см. `health.rs`), этот порт публикуется наружу
|
||||
//! и ОБЯЗАТЕЛЬНО должен быть зафайрволен на IP observability-VPS: любой,
|
||||
//! кто до него дотянется, увидит число активных соединений/трафик этой
|
||||
//! ноды (не секрет пользователей, но разведка для DPI/блокировщика).
|
||||
//!
|
||||
//! Тот же hand-rolled HTTP-паттерн, что и в `health.rs` (не тянуть HTTP-
|
||||
//! фреймворк ради одного эндпоинта на бинарнике, для которого важен размер
|
||||
//! и минимальные зависимости) — просто рендерит текущий снапшот метрик из
|
||||
//! уже установленного `PrometheusHandle` на каждый коннект.
|
||||
|
||||
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
|
||||
use netrunner_logger::{error, info, warn};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
/// Регистрирует глобальный recorder `metrics`-крейта — вызывать ровно один
|
||||
/// раз при старте, до первого `metrics::counter!`/`gauge!`/`histogram!`.
|
||||
pub fn install_recorder() -> PrometheusHandle {
|
||||
PrometheusBuilder::new()
|
||||
.install_recorder()
|
||||
.expect("Failed to install Prometheus recorder")
|
||||
}
|
||||
|
||||
pub async fn run(host: String, port: u16, handle: PrometheusHandle, token: CancellationToken) {
|
||||
let addr = format!("{host}:{port}");
|
||||
let listener = match TcpListener::bind(&addr).await {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
error!(error = %e, addr = %addr, "Не удалось поднять metrics-листенер");
|
||||
return;
|
||||
}
|
||||
};
|
||||
info!("📊 Metrics-эндпоинт слушает на {}", addr);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = token.cancelled() => {
|
||||
info!("Metrics-listener остановлен по сигналу отмены.");
|
||||
break;
|
||||
}
|
||||
res = listener.accept() => {
|
||||
let Ok((mut stream, _)) = res else { continue };
|
||||
let body = handle.render();
|
||||
tokio::spawn(async move {
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
if let Err(e) = stream.write_all(response.as_bytes()).await {
|
||||
warn!(error = %e, "Metrics-listener: ошибка записи ответа");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -175,7 +175,9 @@ impl Network {
|
||||
self.auth.clone(),
|
||||
);
|
||||
|
||||
active_connections.fetch_add(1, Ordering::Relaxed);
|
||||
let active_now = active_connections.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
metrics::gauge!("netrunner_connections_active").set(active_now as f64);
|
||||
metrics::counter!("netrunner_connections_total").increment(1);
|
||||
let conn_counter = active_connections.clone();
|
||||
tokio::spawn(async move {
|
||||
// "Входим" в этот Span. Все логи внутри handler.run() привяжутся к этому IP.
|
||||
@@ -185,7 +187,8 @@ impl Network {
|
||||
if let Err(e) = handler.run().await {
|
||||
error!(error = %e, "⚠️ Server handler terminated with error");
|
||||
}
|
||||
conn_counter.fetch_sub(1, Ordering::Relaxed);
|
||||
let active_now = conn_counter.fetch_sub(1, Ordering::Relaxed) - 1;
|
||||
metrics::gauge!("netrunner_connections_active").set(active_now as f64);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -207,6 +210,9 @@ impl Network {
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
info!("✅ Drain завершён, соединений осталось: {}", active_connections.load(Ordering::Relaxed));
|
||||
info!(
|
||||
"✅ Drain завершён, соединений осталось: {}",
|
||||
active_connections.load(Ordering::Relaxed)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user