116 lines
4.3 KiB
Python
116 lines
4.3 KiB
Python
"""Connection state machine + reconnect backoff (SPECIFICATION §10.2).
|
||
|
||
States: Connecting → Connected → ReconnectWait → Connecting …
|
||
Backoff is exponential 1 s → 2 s → … capped at 60 s; every disconnect/reconnect
|
||
pair is an event (D1/D2) with the outage duration computed and reported.
|
||
|
||
The engine is pure: it is fed timestamps and returns transitions plus events,
|
||
so the poller owns the actual clock/sleeping and tests drive it deterministically.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime, timedelta
|
||
from enum import Enum
|
||
|
||
|
||
class ConnectionState(Enum):
|
||
"""States of the connection state machine (SPECIFICATION §10.2)."""
|
||
|
||
CONNECTING = "connecting"
|
||
CONNECTED = "connected"
|
||
RECONNECT_WAIT = "reconnect_wait"
|
||
|
||
|
||
#: Backoff constants (SPECIFICATION §10.2).
|
||
BACKOFF_INITIAL_S = 1.0
|
||
BACKOFF_MAX_S = 60.0
|
||
|
||
#: D3 reconnect flapping: > this many disconnects within the window is CRITICAL.
|
||
FLAPPING_DISCONNECTS = 3
|
||
FLAPPING_WINDOW = timedelta(minutes=10)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class StateEvent:
|
||
"""One state-machine transition worth reporting (D1/D2/D3)."""
|
||
|
||
kind: str # "disconnected" | "reconnected" | "flapping"
|
||
at: datetime
|
||
detail: dict = field(default_factory=dict)
|
||
|
||
|
||
@dataclass
|
||
class ConnectionStateMachine:
|
||
"""Tracks connection state, backoff and reconnect accounting (D1–D3).
|
||
|
||
The poller calls :meth:`on_connected` / :meth:`on_disconnected` at the
|
||
moments those things are observed; the machine decides *when* to emit
|
||
events (D1 with the failure reason, D2 with the outage duration, D3 when
|
||
disconnects flap) and *how long* to wait before the next attempt.
|
||
"""
|
||
|
||
state: ConnectionState = ConnectionState.CONNECTING
|
||
backoff_s: float = BACKOFF_INITIAL_S
|
||
connected_since: datetime | None = None
|
||
disconnected_since: datetime | None = None
|
||
reconnects_total: int = 0
|
||
_disconnect_times: list[datetime] = field(default_factory=list)
|
||
|
||
def on_connected(self, at: datetime) -> list[StateEvent]:
|
||
"""A connect (or reconnect) succeeded at ``at``."""
|
||
events: list[StateEvent] = []
|
||
was_outage = self.state is ConnectionState.RECONNECT_WAIT
|
||
self.state = ConnectionState.CONNECTED
|
||
self.connected_since = at
|
||
self.backoff_s = BACKOFF_INITIAL_S # reset on success (spec §10.2)
|
||
if was_outage and self.disconnected_since is not None:
|
||
self.reconnects_total += 1
|
||
events.append(
|
||
StateEvent(
|
||
kind="reconnected",
|
||
at=at,
|
||
detail={
|
||
"outage_s": (at - self.disconnected_since).total_seconds(),
|
||
"reconnects_total": self.reconnects_total,
|
||
},
|
||
)
|
||
)
|
||
self.disconnected_since = None
|
||
return events
|
||
|
||
def on_disconnected(self, at: datetime, reason: str) -> list[StateEvent]:
|
||
"""The connection dropped at ``at`` (reason = decoded failure)."""
|
||
events: list[StateEvent] = []
|
||
first_drop = self.state is not ConnectionState.RECONNECT_WAIT
|
||
self.state = ConnectionState.RECONNECT_WAIT
|
||
self.connected_since = None
|
||
if first_drop:
|
||
self.disconnected_since = at
|
||
self._disconnect_times.append(at)
|
||
self._disconnect_times = [
|
||
t for t in self._disconnect_times if at - t <= FLAPPING_WINDOW
|
||
]
|
||
events.append(StateEvent(kind="disconnected", at=at, detail={"reason": reason}))
|
||
if len(self._disconnect_times) > FLAPPING_DISCONNECTS:
|
||
events.append(
|
||
StateEvent(
|
||
kind="flapping",
|
||
at=at,
|
||
detail={"disconnects_in_window": len(self._disconnect_times)},
|
||
)
|
||
)
|
||
return events
|
||
|
||
def next_attempt_delay_s(self) -> float:
|
||
"""How long to wait before the next connect attempt (exponential backoff).
|
||
|
||
Call once per failed attempt; the value doubles up to
|
||
:data:`BACKOFF_MAX_S` and resets only on a successful connect.
|
||
"""
|
||
if self.state is not ConnectionState.RECONNECT_WAIT:
|
||
return 0.0
|
||
delay = self.backoff_s
|
||
self.backoff_s = min(self.backoff_s * 2, BACKOFF_MAX_S)
|
||
return delay |