fix: stabilize tunnel downloads (deadlock, stuck-consumer eviction, real diagnostics)

Downloads were dying and stalling under real network conditions. Root causes
fixed in the muxer's stream-delivery path:

- dispatch_to_local held a DashMap Ref across a nested remove_stream call,
  self-deadlocking the leg's reader task whenever a stream's backlog needed
  eviction — permanently losing a tokio worker thread per occurrence.
- Eviction now runs off the hot path entirely, in a periodic background
  reaper, and only fires when a stream is both over its byte budget AND has
  made no delivery progress for an RTT-adaptive grace window (was a flat
  300ms then 5s, which killed merely-slow-but-alive consumers).
- The server never told the client when a bridge ended normally (only on
  failed connect), leaving the client's virtual TCP socket stuck in
  CloseWait for a full idle timeout and slowing the whole engine loop.
- The client's upload path silently dropped in-flight chunks when all tunnel
  legs were briefly down instead of pausing and retrying like the server
  bridge already did.
- Tried end-to-end credit-based flow control for the download producer;
  reverted the enforcement (kept the dormant frame/API) after it proved
  worse than the pre-existing local channel backpressure — RTT-coupled
  pauses produced burst-then-stall downloads and jitter/ping spikes on the
  shared physical leg.
- Server-side diagnostics snapshot was a stub (always-empty tunnel state,
  misleading placeholder trigger); now aggregates real per-session Muxer
  metrics from the SessionManager.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Kirill
2026-07-02 15:06:00 +07:00
parent 582ad714ae
commit 3b2296a6d9
13 changed files with 693 additions and 136 deletions
+51 -19
View File
@@ -5,12 +5,15 @@
//! событие) с ротацией файла по размеру. Дополнительно пишет стартовый и
//! периодические heartbeat-снапшоты, чтобы файл всегда существовал и было видно,
//! что сервер жив. У сервера нет smoltcp-движка, поэтому socket-метрики пусты —
//! только метрики туннеля.
//! только метрики туннеля, но они реальные: логгер держит [`SessionManager`] и
//! на каждый снапшот опрашивает `Muxer` всех живых сессий (см.
//! [`ServerDiagnosticsLogger::snapshot_all_sessions`]).
use netrunner_core::net::diagnostics::{
self, DiagnosticsEvent, DiagnosticsSnapshot, DiagnosticsStore, TunnelMetrics,
current_timestamp_ms,
};
use netrunner_core::net::SessionManager;
use netrunner_logger::{error, info, warn};
use std::{path::PathBuf, sync::Arc};
use tokio::{
@@ -34,15 +37,19 @@ const HEARTBEAT_INTERVAL_SECS: u64 = 60;
pub struct ServerDiagnosticsLogger {
store: Arc<DiagnosticsStore>,
log_path: PathBuf,
/// Общий реестр сессий сервера — источник правды для реальных метрик
/// туннеля (раньше их не было вовсе, снапшот всегда сообщал пустоту).
session_manager: Arc<SessionManager>,
}
impl ServerDiagnosticsLogger {
pub fn new(log_dir: impl Into<PathBuf>) -> Self {
pub fn new(log_dir: impl Into<PathBuf>, session_manager: Arc<SessionManager>) -> Self {
let mut path = log_dir.into();
path.push("netrunner_diagnostics.jsonl");
Self {
store: Arc::new(DiagnosticsStore::new(SERVER_MAX_SNAPSHOTS)),
log_path: path,
session_manager,
}
}
@@ -61,10 +68,9 @@ impl ServerDiagnosticsLogger {
);
// Write an immediate startup snapshot so the file is created on boot.
let startup = logger.build_server_snapshot(DiagnosticsEvent::LegReconnecting {
leg_id: 0,
attempt: 0,
}).await;
let startup = logger
.build_server_snapshot(DiagnosticsEvent::Heartbeat)
.await;
logger.store.push(startup.clone());
logger.append_to_file(&startup).await;
@@ -86,10 +92,9 @@ impl ServerDiagnosticsLogger {
_ = heartbeat.tick() => {
// Periodic heartbeat — lets operators confirm the server is
// alive even when everything is working perfectly.
let snap = logger.build_server_snapshot(DiagnosticsEvent::LegReconnecting {
leg_id: 0,
attempt: 0,
}).await;
let snap = logger
.build_server_snapshot(DiagnosticsEvent::Heartbeat)
.await;
logger.store.push(snap.clone());
logger.append_to_file(&snap).await;
}
@@ -104,25 +109,52 @@ impl ServerDiagnosticsLogger {
}
/// Builds a server-side snapshot. The server has no smoltcp engine, so
/// socket metrics are omitted; only tunnel metrics are included.
/// socket metrics are omitted; tunnel metrics are real, aggregated across
/// every currently connected client session.
async fn build_server_snapshot(&self, trigger: DiagnosticsEvent) -> DiagnosticsSnapshot {
DiagnosticsSnapshot {
timestamp_ms: current_timestamp_ms(),
trigger,
engine: None,
// Tunnel metrics can be added later if the server gains access to
// a specific Muxer. For now we report an empty structure.
tunnel: TunnelMetrics {
global_min_rtt_ms: netrunner_core::net::GLOBAL_MIN_RTT
.load(std::sync::atomic::Ordering::Relaxed),
active_legs: vec![],
total_streams: 0,
},
tunnel: self.snapshot_all_sessions(),
sockets: vec![],
error_totals: diagnostics::DIAG_COUNTERS.snapshot(),
}
}
/// Опрашивает `Muxer` каждой живой сессии и сводит их в одну [`TunnelMetrics`]:
/// ноги всех сессий (каждая помечена своим `session_id` — иначе `leg_id`
/// разных клиентов совпадали бы, они нумеруются независимо 0..MAX_TUNNEL_LEGS)
/// и сумма потоков. `global_min_rtt_ms` остаётся процесс-глобальным значением
/// ([`GLOBAL_MIN_RTT`](netrunner_core::net::GLOBAL_MIN_RTT)) — это отдельное,
/// более глубокое ограничение (RTT не разведён по сессиям нигде в ядре), не
/// то же самое, что пустая заглушка активных ног/потоков, которую эта функция
/// заменяет.
fn snapshot_all_sessions(&self) -> TunnelMetrics {
let sessions: Vec<_> = self
.session_manager
.get_session()
.iter()
.map(|entry| entry.value().clone())
.collect();
let mut active_legs = Vec::new();
let mut total_streams = 0;
for muxer in &sessions {
let m = muxer.snapshot_tunnel_metrics();
total_streams += m.total_streams;
active_legs.extend(m.active_legs);
}
TunnelMetrics {
global_min_rtt_ms: netrunner_core::net::GLOBAL_MIN_RTT
.load(std::sync::atomic::Ordering::Relaxed),
active_legs,
total_streams,
session_count: sessions.len(),
}
}
/// Appends one JSON line to the log file, rotating if the file is too large.
async fn append_to_file(&self, snap: &DiagnosticsSnapshot) {
if let Err(e) = self.try_rotate().await {