292 lines
12 KiB
Python
292 lines
12 KiB
Python
"""The poll loop on the single dedicated COM thread (SPECIFICATION §10.3/§10.4).
|
||
|
||
Architecture (all COM stays on the worker thread — SPECIFICATION §10.3):
|
||
|
||
- :meth:`Poller.run_forever` runs on the *caller's* thread as the coordinator:
|
||
it starts a worker per (re)connect session, consumes its queue, feeds the
|
||
state machine, applies backoff, and owns the stop flag.
|
||
- The worker thread builds its **own** client via ``client_factory`` (COM
|
||
initialization is per-thread — the client must be born there), polls every
|
||
``interval_s``, measures RTT, and pushes messages onto the queue.
|
||
- Hang watchdog (SPECIFICATION §10.4): the coordinator timestamps every
|
||
worker message; silence longer than ``hard_hang_s`` means the thread is
|
||
stuck in a dead RPC channel — it is **abandoned** (daemon thread, never
|
||
joined) and a fresh worker is built after D1/backoff. A poll that merely
|
||
exceeds ``timeout_ms`` raises the D7 soft-hang event but keeps the thread.
|
||
|
||
Messages the worker sends: ``("connected", ts)``, ``("outcome", PollOutcome)``,
|
||
``("failed", reason)`` — after which the worker exits its loop.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import queue
|
||
import threading
|
||
import time
|
||
from collections.abc import Callable
|
||
from datetime import datetime, timezone
|
||
from typing import Any
|
||
|
||
from opcda_diagnozer.config import Config
|
||
from opcda_diagnozer.events import DiagEvent, EventContext, EventKind, Severity
|
||
from opcda_diagnozer.monitor.detectors import Detection, DetectorSuite, PollOutcome
|
||
from opcda_diagnozer.monitor.state import (
|
||
ConnectionState,
|
||
ConnectionStateMachine,
|
||
StateEvent,
|
||
)
|
||
from opcda_diagnozer.opc.base import OpcClient, OpcError
|
||
|
||
#: Coordinator queue poll slice: small enough that stop is snappy, large
|
||
#: enough to not busy-spin (hard_hang watching happens between slices).
|
||
_WATCH_SLICE_S = 0.25
|
||
|
||
_MESSAGE = tuple[str, Any]
|
||
|
||
|
||
def _utc_now() -> datetime:
|
||
"""Production wall clock (injected as ``clock`` in tests)."""
|
||
return datetime.now(timezone.utc)
|
||
|
||
|
||
class Poller:
|
||
"""Drives one :class:`OpcClient` per connection on its own thread, forever.
|
||
|
||
``client_factory`` is called **on the worker thread** for every
|
||
(re)connect — the client is thread-confined by construction (spec §10.3).
|
||
``reporter`` receives every event; the coordinator never touches the
|
||
client object.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
client_factory: Callable[[], OpcClient],
|
||
config: Config,
|
||
reporter: Callable[[DiagEvent], None],
|
||
clock: Callable[[], datetime] = _utc_now,
|
||
stop: threading.Event | None = None,
|
||
) -> None:
|
||
self._client_factory = client_factory
|
||
self._config = config
|
||
self._reporter = reporter
|
||
self._clock = clock
|
||
self._stop = stop if stop is not None else threading.Event()
|
||
self._sm = ConnectionStateMachine()
|
||
self._suite = DetectorSuite(config)
|
||
self._started_at: datetime | None = None
|
||
# Surveillance tags + device-indicator tags (polled automatically, §6.1),
|
||
# deduplicated in order.
|
||
self._tags: tuple[str, ...] = tuple(
|
||
dict.fromkeys(
|
||
[t.name for t in config.tags] + [d.tag for d in config.devices]
|
||
)
|
||
)
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# public surface
|
||
# ------------------------------------------------------------------ #
|
||
|
||
def request_stop(self) -> None:
|
||
"""Ask the loop to finish after the current poll (from any thread)."""
|
||
self._stop.set()
|
||
|
||
def run_forever(self) -> None:
|
||
"""Coordinate worker sessions until :meth:`request_stop`.
|
||
|
||
Never raises for COM/OPC failures — they are events (D1/D2). A worker
|
||
crash surfaces as DISCONNECTED with the exception text, then reconnect.
|
||
"""
|
||
self._started_at = self._clock()
|
||
last_heartbeat = self._started_at
|
||
while not self._stop.is_set():
|
||
outcomes: queue.Queue[_MESSAGE] = queue.Queue()
|
||
worker = threading.Thread(
|
||
target=self._worker_main, args=(outcomes,), daemon=True
|
||
)
|
||
worker.start()
|
||
hung = self._coordinate(outcomes)
|
||
if hung:
|
||
# SPECIFICATION §10.4: abandon a thread stuck in a dead RPC
|
||
# channel — never join it; daemon=True lets the process exit.
|
||
self._emit(
|
||
EventKind.POLL_HANG,
|
||
Severity.CRITICAL,
|
||
{
|
||
"hard_hang_s": self._config.poll.hard_hang_s,
|
||
"action": "COM thread abandoned and rebuilt",
|
||
},
|
||
)
|
||
else:
|
||
worker.join(timeout=self._config.poll.hard_hang_s)
|
||
if self._stop.is_set():
|
||
break
|
||
if self._sm.state is ConnectionState.RECONNECT_WAIT:
|
||
delay = self._sm.next_attempt_delay_s()
|
||
if delay:
|
||
self._stop.wait(delay)
|
||
self._emit(
|
||
EventKind.DISCONNECTED,
|
||
Severity.INFO,
|
||
{"reason": "shutting down (stop requested)"},
|
||
)
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# coordinator side (caller's thread)
|
||
# ------------------------------------------------------------------ #
|
||
|
||
def _coordinate(self, outcomes: "queue.Queue[_MESSAGE]") -> bool:
|
||
"""Consume one worker session; returns True when the worker was hung.
|
||
|
||
Feeds the state machine and detectors, emits D1/D2/D7/D15/D4–D6/D8–D14/D16
|
||
events, and watches for hard hangs by wall-clock silence.
|
||
"""
|
||
last_worker_word = self._clock()
|
||
last_heartbeat = self._started_at or self._clock()
|
||
while not self._stop.is_set():
|
||
try:
|
||
kind, payload = outcomes.get(timeout=_WATCH_SLICE_S)
|
||
except queue.Empty:
|
||
silent_s = (self._clock() - last_worker_word).total_seconds()
|
||
if silent_s > self._config.poll.hard_hang_s:
|
||
return True
|
||
continue
|
||
last_worker_word = self._clock()
|
||
if kind == "connected":
|
||
for ev in self._sm.on_connected(payload):
|
||
self._emit_state_event(ev)
|
||
self._emit(
|
||
EventKind.CONNECTED,
|
||
Severity.INFO,
|
||
{"server": self._config.opc.server},
|
||
)
|
||
elif kind == "outcome":
|
||
outcome: PollOutcome = payload
|
||
for event in self._consume_outcome(outcome):
|
||
self._reporter(event)
|
||
now = outcome.started_at
|
||
if (now - last_heartbeat).total_seconds() >= self._config.report.heartbeat_s:
|
||
last_heartbeat = now
|
||
self._emit(EventKind.HEARTBEAT, Severity.INFO, self._heartbeat_detail())
|
||
elif kind == "failed":
|
||
for ev in self._sm.on_disconnected(self._clock(), reason=str(payload)):
|
||
self._emit_state_event(ev)
|
||
return False
|
||
return False
|
||
|
||
def _consume_outcome(self, outcome: PollOutcome) -> list[DiagEvent]:
|
||
"""Run the detector suite on one outcome; add D7 soft-hang; map to events."""
|
||
events: list[DiagEvent] = []
|
||
for detection in self._suite.observe(outcome):
|
||
events.append(self._to_event(detection))
|
||
if outcome.error is None and outcome.rtt_ms > self._config.poll.timeout_ms:
|
||
events.append(
|
||
self._to_event(
|
||
Detection(
|
||
kind=EventKind.POLL_HANG,
|
||
severity=Severity.CRITICAL,
|
||
detail={
|
||
"rtt_ms": round(outcome.rtt_ms, 1),
|
||
"timeout_ms": self._config.poll.timeout_ms,
|
||
"note": "call returned late; thread kept",
|
||
},
|
||
)
|
||
)
|
||
)
|
||
return events
|
||
|
||
def _to_event(self, detection: Detection) -> DiagEvent:
|
||
return DiagEvent(
|
||
time_utc=self._clock(),
|
||
kind=detection.kind,
|
||
severity=detection.severity,
|
||
detail=detection.detail,
|
||
context=EventContext(
|
||
uptime_s=self._uptime_s(),
|
||
reconnects_total=self._sm.reconnects_total,
|
||
rtt_p95_ms=self._suite.rtt_p95_ms(),
|
||
tags_bad=self._suite.tags_bad(),
|
||
tags_total=len(self._tags),
|
||
),
|
||
)
|
||
|
||
def _emit_state_event(self, ev: StateEvent) -> None:
|
||
kind, severity = {
|
||
"disconnected": (EventKind.DISCONNECTED, Severity.CRITICAL),
|
||
"reconnected": (EventKind.RECONNECTED, Severity.INFO),
|
||
"flapping": (EventKind.RECONNECT_FLAPPING, Severity.CRITICAL),
|
||
}[ev.kind]
|
||
self._emit(kind, severity, ev.detail)
|
||
|
||
def _heartbeat_detail(self) -> dict:
|
||
return {
|
||
"uptime_s": self._uptime_s(),
|
||
"reconnects_total": self._sm.reconnects_total,
|
||
"rtt_p95_ms": self._suite.rtt_p95_ms(),
|
||
"tags_bad": self._suite.tags_bad(),
|
||
"tags_total": len(self._tags),
|
||
"devices_down": self._suite.devices_down(),
|
||
}
|
||
|
||
def _uptime_s(self) -> float:
|
||
if self._started_at is None:
|
||
return 0.0
|
||
return (self._clock() - self._started_at).total_seconds()
|
||
|
||
def _emit(self, kind: EventKind, severity: Severity, detail: dict) -> None:
|
||
self._reporter(self._to_event(Detection(kind=kind, severity=severity, detail=detail)))
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# worker side (the dedicated COM thread)
|
||
# ------------------------------------------------------------------ #
|
||
|
||
def _worker_main(self, outcomes: "queue.Queue[_MESSAGE]") -> None:
|
||
"""One connection session: own client, poll loop, push messages.
|
||
|
||
Exits (putting ``("failed", reason)``) whenever the connection drops
|
||
or the stop flag is observed — the coordinator decides what next.
|
||
"""
|
||
client: OpcClient | None = None
|
||
try:
|
||
client = self._client_factory()
|
||
client.connect()
|
||
outcomes.put(("connected", self._clock()))
|
||
while not self._stop.is_set():
|
||
started = time.monotonic()
|
||
started_at = _utc_now()
|
||
try:
|
||
results = client.read(self._tags)
|
||
rtt_ms = (time.monotonic() - started) * 1000.0
|
||
outcomes.put(
|
||
(
|
||
"outcome",
|
||
PollOutcome(
|
||
started_at=started_at,
|
||
rtt_ms=rtt_ms,
|
||
results=tuple(results),
|
||
),
|
||
)
|
||
)
|
||
except OpcError as exc:
|
||
rtt_ms = (time.monotonic() - started) * 1000.0
|
||
outcomes.put(
|
||
(
|
||
"outcome",
|
||
PollOutcome(
|
||
started_at=started_at, rtt_ms=rtt_ms, error=str(exc)
|
||
),
|
||
)
|
||
)
|
||
outcomes.put(("failed", str(exc)))
|
||
return
|
||
if self._stop.wait(self._config.poll.interval_s):
|
||
break
|
||
except OpcError as exc:
|
||
outcomes.put(("failed", str(exc)))
|
||
except Exception as exc: # noqa: BLE001 — a worker death is an event, not a crash
|
||
outcomes.put(("failed", f"worker crashed: {type(exc).__name__}: {exc}"))
|
||
finally:
|
||
if client is not None:
|
||
try:
|
||
client.close()
|
||
except Exception: # noqa: BLE001 — best-effort teardown
|
||
pass |