"""The poll loop on the single dedicated COM thread (SPECIFICATION §10.3/§10.4). Architecture (all COM stays on the worker thread — SPECIFICATION §10.3): - :meth:`Poller.run_forever` runs on the *caller's* thread as the coordinator: it starts a worker per (re)connect session, consumes its queue, feeds the state machine, applies backoff, and owns the stop flag. - The worker thread builds its **own** client via ``client_factory`` (COM initialization is per-thread — the client must be born there), polls every ``interval_s``, measures RTT, and pushes messages onto the queue. - Hang watchdog (SPECIFICATION §10.4): the coordinator timestamps every worker message; silence longer than ``hard_hang_s`` means the thread is stuck in a dead RPC channel — it is **abandoned** (daemon thread, never joined) and a fresh worker is built after D1/backoff. A poll that merely exceeds ``timeout_ms`` raises the D7 soft-hang event but keeps the thread. Messages the worker sends: ``("connected", ts)``, ``("outcome", PollOutcome)``, ``("failed", reason)`` — after which the worker exits its loop. """ from __future__ import annotations import queue import threading import time from collections.abc import Callable from datetime import datetime, timezone from typing import Any from opcda_diagnozer.config import Config from opcda_diagnozer.events import DiagEvent, EventContext, EventKind, Severity from opcda_diagnozer.monitor.detectors import Detection, DetectorSuite, PollOutcome from opcda_diagnozer.monitor.state import ( ConnectionState, ConnectionStateMachine, StateEvent, ) from opcda_diagnozer.opc.base import OpcClient, OpcError #: Coordinator queue poll slice: small enough that stop is snappy, large #: enough to not busy-spin (hard_hang watching happens between slices). _WATCH_SLICE_S = 0.25 _MESSAGE = tuple[str, Any] def _utc_now() -> datetime: """Production wall clock (injected as ``clock`` in tests).""" return datetime.now(timezone.utc) class Poller: """Drives one :class:`OpcClient` per connection on its own thread, forever. ``client_factory`` is called **on the worker thread** for every (re)connect — the client is thread-confined by construction (spec §10.3). ``reporter`` receives every event; the coordinator never touches the client object. """ def __init__( self, client_factory: Callable[[], OpcClient], config: Config, reporter: Callable[[DiagEvent], None], clock: Callable[[], datetime] = _utc_now, stop: threading.Event | None = None, ) -> None: self._client_factory = client_factory self._config = config self._reporter = reporter self._clock = clock self._stop = stop if stop is not None else threading.Event() self._sm = ConnectionStateMachine() self._suite = DetectorSuite(config) self._started_at: datetime | None = None # Surveillance tags + device-indicator tags (polled automatically, §6.1), # deduplicated in order. self._tags: tuple[str, ...] = tuple( dict.fromkeys( [t.name for t in config.tags] + [d.tag for d in config.devices] ) ) # ------------------------------------------------------------------ # # public surface # ------------------------------------------------------------------ # def request_stop(self) -> None: """Ask the loop to finish after the current poll (from any thread).""" self._stop.set() def run_forever(self) -> None: """Coordinate worker sessions until :meth:`request_stop`. Never raises for COM/OPC failures — they are events (D1/D2). A worker crash surfaces as DISCONNECTED with the exception text, then reconnect. """ self._started_at = self._clock() last_heartbeat = self._started_at while not self._stop.is_set(): outcomes: queue.Queue[_MESSAGE] = queue.Queue() worker = threading.Thread( target=self._worker_main, args=(outcomes,), daemon=True ) worker.start() hung = self._coordinate(outcomes) if hung: # SPECIFICATION §10.4: abandon a thread stuck in a dead RPC # channel — never join it; daemon=True lets the process exit. self._emit( EventKind.POLL_HANG, Severity.CRITICAL, { "hard_hang_s": self._config.poll.hard_hang_s, "action": "COM thread abandoned and rebuilt", }, ) else: worker.join(timeout=self._config.poll.hard_hang_s) if self._stop.is_set(): break if self._sm.state is ConnectionState.RECONNECT_WAIT: delay = self._sm.next_attempt_delay_s() if delay: self._stop.wait(delay) self._emit( EventKind.DISCONNECTED, Severity.INFO, {"reason": "shutting down (stop requested)"}, ) # ------------------------------------------------------------------ # # coordinator side (caller's thread) # ------------------------------------------------------------------ # def _coordinate(self, outcomes: "queue.Queue[_MESSAGE]") -> bool: """Consume one worker session; returns True when the worker was hung. Feeds the state machine and detectors, emits D1/D2/D7/D15/D4–D6/D8–D14/D16 events, and watches for hard hangs by wall-clock silence. """ last_worker_word = self._clock() last_heartbeat = self._started_at or self._clock() while not self._stop.is_set(): try: kind, payload = outcomes.get(timeout=_WATCH_SLICE_S) except queue.Empty: silent_s = (self._clock() - last_worker_word).total_seconds() if silent_s > self._config.poll.hard_hang_s: return True continue last_worker_word = self._clock() if kind == "connected": for ev in self._sm.on_connected(payload): self._emit_state_event(ev) self._emit( EventKind.CONNECTED, Severity.INFO, {"server": self._config.opc.server}, ) elif kind == "outcome": outcome: PollOutcome = payload for event in self._consume_outcome(outcome): self._reporter(event) now = outcome.started_at if (now - last_heartbeat).total_seconds() >= self._config.report.heartbeat_s: last_heartbeat = now self._emit(EventKind.HEARTBEAT, Severity.INFO, self._heartbeat_detail()) elif kind == "failed": for ev in self._sm.on_disconnected(self._clock(), reason=str(payload)): self._emit_state_event(ev) return False return False def _consume_outcome(self, outcome: PollOutcome) -> list[DiagEvent]: """Run the detector suite on one outcome; add D7 soft-hang; map to events.""" events: list[DiagEvent] = [] for detection in self._suite.observe(outcome): events.append(self._to_event(detection)) if outcome.error is None and outcome.rtt_ms > self._config.poll.timeout_ms: events.append( self._to_event( Detection( kind=EventKind.POLL_HANG, severity=Severity.CRITICAL, detail={ "rtt_ms": round(outcome.rtt_ms, 1), "timeout_ms": self._config.poll.timeout_ms, "note": "call returned late; thread kept", }, ) ) ) return events def _to_event(self, detection: Detection) -> DiagEvent: return DiagEvent( time_utc=self._clock(), kind=detection.kind, severity=detection.severity, detail=detection.detail, context=EventContext( uptime_s=self._uptime_s(), reconnects_total=self._sm.reconnects_total, rtt_p95_ms=self._suite.rtt_p95_ms(), tags_bad=self._suite.tags_bad(), tags_total=len(self._tags), ), ) def _emit_state_event(self, ev: StateEvent) -> None: kind, severity = { "disconnected": (EventKind.DISCONNECTED, Severity.CRITICAL), "reconnected": (EventKind.RECONNECTED, Severity.INFO), "flapping": (EventKind.RECONNECT_FLAPPING, Severity.CRITICAL), }[ev.kind] self._emit(kind, severity, ev.detail) def _heartbeat_detail(self) -> dict: return { "uptime_s": self._uptime_s(), "reconnects_total": self._sm.reconnects_total, "rtt_p95_ms": self._suite.rtt_p95_ms(), "tags_bad": self._suite.tags_bad(), "tags_total": len(self._tags), "devices_down": self._suite.devices_down(), } def _uptime_s(self) -> float: if self._started_at is None: return 0.0 return (self._clock() - self._started_at).total_seconds() def _emit(self, kind: EventKind, severity: Severity, detail: dict) -> None: self._reporter(self._to_event(Detection(kind=kind, severity=severity, detail=detail))) # ------------------------------------------------------------------ # # worker side (the dedicated COM thread) # ------------------------------------------------------------------ # def _worker_main(self, outcomes: "queue.Queue[_MESSAGE]") -> None: """One connection session: own client, poll loop, push messages. Exits (putting ``("failed", reason)``) whenever the connection drops or the stop flag is observed — the coordinator decides what next. """ client: OpcClient | None = None try: client = self._client_factory() client.connect() outcomes.put(("connected", self._clock())) while not self._stop.is_set(): started = time.monotonic() started_at = _utc_now() try: results = client.read(self._tags) rtt_ms = (time.monotonic() - started) * 1000.0 outcomes.put( ( "outcome", PollOutcome( started_at=started_at, rtt_ms=rtt_ms, results=tuple(results), ), ) ) except OpcError as exc: rtt_ms = (time.monotonic() - started) * 1000.0 outcomes.put( ( "outcome", PollOutcome( started_at=started_at, rtt_ms=rtt_ms, error=str(exc) ), ) ) outcomes.put(("failed", str(exc))) return if self._stop.wait(self._config.poll.interval_s): break except OpcError as exc: outcomes.put(("failed", str(exc))) except Exception as exc: # noqa: BLE001 — a worker death is an event, not a crash outcomes.put(("failed", f"worker crashed: {type(exc).__name__}: {exc}")) finally: if client is not None: try: client.close() except Exception: # noqa: BLE001 — best-effort teardown pass