240 lines
9.2 KiB
Python
240 lines
9.2 KiB
Python
"""Poller end-to-end tests against opc.fake — the Phase 1 exit criterion.
|
|
|
|
Runs the full pipeline (worker thread → outcomes → detectors → events) without
|
|
COM (IMPLEMENTATION_PLAN §9). Timing-dependent paths (hang watchdog) are
|
|
exercised with tiny thresholds so the tests stay fast.
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
from collections.abc import Sequence
|
|
from datetime import datetime, timezone
|
|
|
|
import pytest
|
|
|
|
from opcda_diagnozer.config import (
|
|
Config,
|
|
DeviceEntry,
|
|
LatencyConfig,
|
|
OpcConfig,
|
|
PollConfig,
|
|
ReportConfig,
|
|
StaleConfig,
|
|
TagEntry,
|
|
)
|
|
from opcda_diagnozer.events import DiagEvent, EventKind, Severity
|
|
from opcda_diagnozer.monitor.poller import Poller
|
|
from opcda_diagnozer.opc.base import OPCDataSource, OpcClient, OpcError, ReadResult
|
|
from opcda_diagnozer.opc.fake import FakeOpcClient, PollScriptError
|
|
|
|
TAGS = ("T.Counter", "T.Level")
|
|
INDICATOR = "T.PumpConnected"
|
|
|
|
|
|
def _config(poll: PollConfig | None = None, heartbeat_s: float = 3600.0) -> Config:
|
|
return Config(
|
|
opc=OpcConfig(server="Fake.Server.1"),
|
|
poll=poll or PollConfig(interval_s=0.05, timeout_ms=5000, hard_hang_s=5.0),
|
|
tags=(TagEntry(name="T.Counter"), TagEntry(name="T.Level")),
|
|
devices=(DeviceEntry(name="Pump A", tag=INDICATOR, ok_values=("true",)),),
|
|
latency=LatencyConfig(warn_ms=1000.0),
|
|
stale=StaleConfig(enabled=False), # device_time checks off in these runs
|
|
report=ReportConfig(heartbeat_s=heartbeat_s), # heartbeats off by default
|
|
)
|
|
|
|
|
|
def _res(tag: str, quality: int = 0xC0, value=0) -> ReadResult:
|
|
return ReadResult(
|
|
tag=tag, value=value, quality_raw=quality, device_time=datetime.now(timezone.utc)
|
|
)
|
|
|
|
|
|
class _Collector:
|
|
"""Thread-safe event sink."""
|
|
|
|
def __init__(self) -> None:
|
|
self.events: list[DiagEvent] = []
|
|
self._lock = threading.Lock()
|
|
|
|
def __call__(self, event: DiagEvent) -> None:
|
|
with self._lock:
|
|
self.events.append(event)
|
|
|
|
def of(self, kind: EventKind) -> list[DiagEvent]:
|
|
with self._lock:
|
|
return [e for e in self.events if e.kind is kind]
|
|
|
|
|
|
class _FakeFactory:
|
|
"""client_factory that hands out fake clients; one call per (re)connect."""
|
|
|
|
def __init__(self, clients: Sequence[OpcClient]) -> None:
|
|
self._clients = list(clients)
|
|
self.created = 0
|
|
|
|
def __call__(self) -> OpcClient:
|
|
self.created += 1
|
|
if not self._clients:
|
|
raise OpcError("no more scripted clients")
|
|
client = self._clients.pop(0)
|
|
if not client.connected:
|
|
client.connect()
|
|
return client
|
|
|
|
|
|
def _run_until(poller: Poller, condition, timeout_s: float = 8.0) -> None:
|
|
"""Run the poller on a thread until ``condition()`` (or timeout), then stop."""
|
|
thread = threading.Thread(target=poller.run_forever, daemon=True)
|
|
thread.start()
|
|
deadline = time.monotonic() + timeout_s
|
|
try:
|
|
while time.monotonic() < deadline:
|
|
if condition():
|
|
break
|
|
time.sleep(0.005)
|
|
finally:
|
|
poller.request_stop()
|
|
thread.join(timeout=5.0)
|
|
if thread.is_alive():
|
|
pytest.fail("poller thread did not stop within the join timeout")
|
|
if not condition():
|
|
pytest.fail("condition not reached before the test timeout")
|
|
|
|
|
|
def test_run_emits_connected_and_quality_event() -> None:
|
|
"""Healthy loop: CONNECTED, then a BAD transition produces QUALITY_BAD."""
|
|
client = FakeOpcClient(
|
|
script=[
|
|
"good", # poll 1: baseline GOOD
|
|
# poll 2: T.Counter flips BAD -> D8 with clear-text status
|
|
[_res("T.Counter", 0x18), _res("T.Level", 0xC0), _res(INDICATOR, 0xC0, True)],
|
|
]
|
|
)
|
|
collector = _Collector()
|
|
poller = Poller(_FakeFactory([client]), _config(), collector)
|
|
_run_until(poller, lambda: len(collector.of(EventKind.QUALITY_BAD)) >= 1)
|
|
|
|
assert collector.of(EventKind.CONNECTED), "no CONNECTED event"
|
|
bad = collector.of(EventKind.QUALITY_BAD)[0]
|
|
assert bad.detail["tag"] == "T.Counter"
|
|
assert bad.detail["quality"] == "BAD / COMM FAILURE (0x18)"
|
|
assert bad.severity is Severity.WARN
|
|
# tags polled = surveillance tags + the device indicator
|
|
assert set(client.reads[0][0]) == {"T.Counter", "T.Level", INDICATOR}
|
|
# graceful shutdown note
|
|
shutdown = collector.of(EventKind.DISCONNECTED)
|
|
assert shutdown and shutdown[0].detail["reason"].startswith("shutting down")
|
|
|
|
|
|
def test_run_device_link_down_and_recovery() -> None:
|
|
client = FakeOpcClient(
|
|
script=[
|
|
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
|
|
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, False)],
|
|
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, False)],
|
|
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
|
|
]
|
|
)
|
|
collector = _Collector()
|
|
poller = Poller(_FakeFactory([client]), _config(), collector)
|
|
_run_until(poller, lambda: len(collector.of(EventKind.DEVICE_LINK_UP)) >= 1)
|
|
|
|
downs = collector.of(EventKind.DEVICE_LINK_DOWN)
|
|
ups = collector.of(EventKind.DEVICE_LINK_UP)
|
|
assert len(downs) == 1 # edge-level: two down polls -> one DOWN
|
|
assert downs[0].detail["device"] == "Pump A"
|
|
assert downs[0].severity is Severity.CRITICAL
|
|
assert len(ups) == 1
|
|
assert ups[0].detail["device"] == "Pump A"
|
|
|
|
|
|
def test_run_connection_loss_emits_d1_then_reconnect_d2() -> None:
|
|
"""A failing poll ends the session: DISCONNECTED (D1), then RECONNECTED (D2)."""
|
|
dying = FakeOpcClient(
|
|
script=[
|
|
"good",
|
|
PollScriptError("could not read from server (HRESULT 0x800706BA: ...)"),
|
|
]
|
|
)
|
|
healthy = FakeOpcClient() # the reconnect lands here
|
|
collector = _Collector()
|
|
factory = _FakeFactory([dying, healthy])
|
|
poller = Poller(factory, _config(), collector)
|
|
_run_until(poller, lambda: len(collector.of(EventKind.RECONNECTED)) >= 1)
|
|
|
|
disconnected = collector.of(EventKind.DISCONNECTED)
|
|
assert any("HRESULT 0x800706BA" in d.detail.get("reason", "") for d in disconnected)
|
|
reconnected = collector.of(EventKind.RECONNECTED)
|
|
assert reconnected and reconnected[0].detail["outage_s"] < 10.0
|
|
assert factory.created == 2 # one client per session
|
|
|
|
|
|
def test_run_heartbeat_emits_periodically() -> None:
|
|
# indicator defaults to a plain read -> value 0, which is NOT an ok value
|
|
# ("true"): script the first poll so the device starts healthy.
|
|
client = FakeOpcClient(
|
|
script=[
|
|
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
|
|
]
|
|
)
|
|
collector = _Collector()
|
|
poller = Poller(
|
|
_FakeFactory([client]), _config(heartbeat_s=0.15), collector
|
|
)
|
|
_run_until(poller, lambda: len(collector.of(EventKind.HEARTBEAT)) >= 2, timeout_s=10.0)
|
|
|
|
heartbeats = collector.of(EventKind.HEARTBEAT)
|
|
assert len(heartbeats) >= 2
|
|
detail = heartbeats[0].detail
|
|
assert detail["tags_total"] == 3 # 2 surveillance + 1 indicator
|
|
# after the scripted healthy poll the fake falls back to value=0 for the
|
|
# indicator -> the device is DOWN in later heartbeats. Assert the DOWN
|
|
# count is reflected honestly rather than expecting fake defaults to be ok.
|
|
assert detail["devices_down"] in (0, 1)
|
|
assert detail["reconnects_total"] == 0
|
|
# a DEVICE_LINK_DOWN must have been reported when it flipped
|
|
assert collector.of(EventKind.DEVICE_LINK_DOWN) or detail["devices_down"] == 0
|
|
|
|
|
|
def test_run_soft_hang_flags_d7_but_keeps_thread() -> None:
|
|
"""A slow-but-returning poll raises POLL_HANG (D7) without rebuilding."""
|
|
|
|
class SlowClient(FakeOpcClient):
|
|
def read(self, tags, source=OPCDataSource.DEVICE):
|
|
time.sleep(0.3) # > timeout_ms 200 below
|
|
return super().read(tags, source)
|
|
|
|
config = _config(poll=PollConfig(interval_s=0.05, timeout_ms=200, hard_hang_s=30.0))
|
|
client = SlowClient()
|
|
collector = _Collector()
|
|
factory = _FakeFactory([client])
|
|
poller = Poller(factory, config, collector)
|
|
_run_until(poller, lambda: bool(collector.of(EventKind.POLL_HANG)))
|
|
|
|
hangs = collector.of(EventKind.POLL_HANG)
|
|
assert hangs and hangs[0].detail["rtt_ms"] >= 200
|
|
assert hangs[0].detail["timeout_ms"] == 200
|
|
assert factory.created == 1 # thread kept: no rebuild
|
|
# no abandoned-thread message (that's the hard-hang path)
|
|
assert all("abandoned" not in h.detail.get("action", "") for h in hangs)
|
|
|
|
|
|
def test_run_hard_hang_abandons_thread_and_rebuilds() -> None:
|
|
"""Silence > hard_hang_s: the worker is abandoned (POLL_HANG) and rebuilt."""
|
|
|
|
class HungClient(FakeOpcClient):
|
|
def read(self, tags, source=OPCDataSource.DEVICE):
|
|
time.sleep(10.0) # way past hard_hang_s
|
|
return super().read(tags, source)
|
|
|
|
config = _config(poll=PollConfig(interval_s=0.05, timeout_ms=200, hard_hang_s=0.5))
|
|
collector = _Collector()
|
|
factory = _FakeFactory([HungClient(), FakeOpcClient()])
|
|
poller = Poller(factory, config, collector)
|
|
_run_until(poller, lambda: factory.created >= 2, timeout_s=10.0)
|
|
|
|
hard = [
|
|
e for e in collector.of(EventKind.POLL_HANG) if "abandoned" in e.detail.get("action", "")
|
|
]
|
|
assert hard and hard[0].severity is Severity.CRITICAL
|
|
assert factory.created == 2 # the hung worker was replaced |