🚧 WIP
This commit is contained in:
Jonas Bengtsson
+229
-22
@@ -1,33 +1,240 @@
|
||||
"""End-to-end pipeline tests against opc.fake (IMPLEMENTATION_PLAN §9).
|
||||
"""Poller end-to-end tests against opc.fake — the Phase 1 exit criterion.
|
||||
|
||||
The fake client lets the event pipeline run without COM/hardware; Phase 1
|
||||
grows this into the full scenario suite.
|
||||
Runs the full pipeline (worker thread → outcomes → detectors → events) without
|
||||
COM (IMPLEMENTATION_PLAN §9). Timing-dependent paths (hang watchdog) are
|
||||
exercised with tiny thresholds so the tests stay fast.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from opcda_diagnozer.opc.base import OpcError
|
||||
from opcda_diagnozer.opc.fake import FakeOpcClient
|
||||
from opcda_diagnozer.config import (
|
||||
Config,
|
||||
DeviceEntry,
|
||||
LatencyConfig,
|
||||
OpcConfig,
|
||||
PollConfig,
|
||||
ReportConfig,
|
||||
StaleConfig,
|
||||
TagEntry,
|
||||
)
|
||||
from opcda_diagnozer.events import DiagEvent, EventKind, Severity
|
||||
from opcda_diagnozer.monitor.poller import Poller
|
||||
from opcda_diagnozer.opc.base import OPCDataSource, OpcClient, OpcError, ReadResult
|
||||
from opcda_diagnozer.opc.fake import FakeOpcClient, PollScriptError
|
||||
|
||||
TAGS = ("T.Counter", "T.Level")
|
||||
INDICATOR = "T.PumpConnected"
|
||||
|
||||
|
||||
def test_fake_reads_good_quality() -> None:
|
||||
client = FakeOpcClient()
|
||||
client.connect()
|
||||
results = client.read(["Channel1.Device1.Counter", "Channel1.Device1.Level"])
|
||||
assert [r.tag for r in results] == ["Channel1.Device1.Counter", "Channel1.Device1.Level"]
|
||||
assert all(r.quality_raw == 0xC0 for r in results)
|
||||
assert all(r.device_time is not None for r in results)
|
||||
def _config(poll: PollConfig | None = None, heartbeat_s: float = 3600.0) -> Config:
|
||||
return Config(
|
||||
opc=OpcConfig(server="Fake.Server.1"),
|
||||
poll=poll or PollConfig(interval_s=0.05, timeout_ms=5000, hard_hang_s=5.0),
|
||||
tags=(TagEntry(name="T.Counter"), TagEntry(name="T.Level")),
|
||||
devices=(DeviceEntry(name="Pump A", tag=INDICATOR, ok_values=("true",)),),
|
||||
latency=LatencyConfig(warn_ms=1000.0),
|
||||
stale=StaleConfig(enabled=False), # device_time checks off in these runs
|
||||
report=ReportConfig(heartbeat_s=heartbeat_s), # heartbeats off by default
|
||||
)
|
||||
|
||||
|
||||
def test_fake_requires_connect_first() -> None:
|
||||
client = FakeOpcClient()
|
||||
with pytest.raises(OpcError):
|
||||
client.read(["Channel1.Device1.Counter"])
|
||||
def _res(tag: str, quality: int = 0xC0, value=0) -> ReadResult:
|
||||
return ReadResult(
|
||||
tag=tag, value=value, quality_raw=quality, device_time=datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
|
||||
def test_fake_close_disconnects() -> None:
|
||||
client = FakeOpcClient()
|
||||
client.connect()
|
||||
client.close()
|
||||
with pytest.raises(OpcError):
|
||||
client.read(["Channel1.Device1.Counter"])
|
||||
class _Collector:
|
||||
"""Thread-safe event sink."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.events: list[DiagEvent] = []
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def __call__(self, event: DiagEvent) -> None:
|
||||
with self._lock:
|
||||
self.events.append(event)
|
||||
|
||||
def of(self, kind: EventKind) -> list[DiagEvent]:
|
||||
with self._lock:
|
||||
return [e for e in self.events if e.kind is kind]
|
||||
|
||||
|
||||
class _FakeFactory:
|
||||
"""client_factory that hands out fake clients; one call per (re)connect."""
|
||||
|
||||
def __init__(self, clients: Sequence[OpcClient]) -> None:
|
||||
self._clients = list(clients)
|
||||
self.created = 0
|
||||
|
||||
def __call__(self) -> OpcClient:
|
||||
self.created += 1
|
||||
if not self._clients:
|
||||
raise OpcError("no more scripted clients")
|
||||
client = self._clients.pop(0)
|
||||
if not client.connected:
|
||||
client.connect()
|
||||
return client
|
||||
|
||||
|
||||
def _run_until(poller: Poller, condition, timeout_s: float = 8.0) -> None:
|
||||
"""Run the poller on a thread until ``condition()`` (or timeout), then stop."""
|
||||
thread = threading.Thread(target=poller.run_forever, daemon=True)
|
||||
thread.start()
|
||||
deadline = time.monotonic() + timeout_s
|
||||
try:
|
||||
while time.monotonic() < deadline:
|
||||
if condition():
|
||||
break
|
||||
time.sleep(0.005)
|
||||
finally:
|
||||
poller.request_stop()
|
||||
thread.join(timeout=5.0)
|
||||
if thread.is_alive():
|
||||
pytest.fail("poller thread did not stop within the join timeout")
|
||||
if not condition():
|
||||
pytest.fail("condition not reached before the test timeout")
|
||||
|
||||
|
||||
def test_run_emits_connected_and_quality_event() -> None:
|
||||
"""Healthy loop: CONNECTED, then a BAD transition produces QUALITY_BAD."""
|
||||
client = FakeOpcClient(
|
||||
script=[
|
||||
"good", # poll 1: baseline GOOD
|
||||
# poll 2: T.Counter flips BAD -> D8 with clear-text status
|
||||
[_res("T.Counter", 0x18), _res("T.Level", 0xC0), _res(INDICATOR, 0xC0, True)],
|
||||
]
|
||||
)
|
||||
collector = _Collector()
|
||||
poller = Poller(_FakeFactory([client]), _config(), collector)
|
||||
_run_until(poller, lambda: len(collector.of(EventKind.QUALITY_BAD)) >= 1)
|
||||
|
||||
assert collector.of(EventKind.CONNECTED), "no CONNECTED event"
|
||||
bad = collector.of(EventKind.QUALITY_BAD)[0]
|
||||
assert bad.detail["tag"] == "T.Counter"
|
||||
assert bad.detail["quality"] == "BAD / COMM FAILURE (0x18)"
|
||||
assert bad.severity is Severity.WARN
|
||||
# tags polled = surveillance tags + the device indicator
|
||||
assert set(client.reads[0][0]) == {"T.Counter", "T.Level", INDICATOR}
|
||||
# graceful shutdown note
|
||||
shutdown = collector.of(EventKind.DISCONNECTED)
|
||||
assert shutdown and shutdown[0].detail["reason"].startswith("shutting down")
|
||||
|
||||
|
||||
def test_run_device_link_down_and_recovery() -> None:
|
||||
client = FakeOpcClient(
|
||||
script=[
|
||||
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
|
||||
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, False)],
|
||||
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, False)],
|
||||
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
|
||||
]
|
||||
)
|
||||
collector = _Collector()
|
||||
poller = Poller(_FakeFactory([client]), _config(), collector)
|
||||
_run_until(poller, lambda: len(collector.of(EventKind.DEVICE_LINK_UP)) >= 1)
|
||||
|
||||
downs = collector.of(EventKind.DEVICE_LINK_DOWN)
|
||||
ups = collector.of(EventKind.DEVICE_LINK_UP)
|
||||
assert len(downs) == 1 # edge-level: two down polls -> one DOWN
|
||||
assert downs[0].detail["device"] == "Pump A"
|
||||
assert downs[0].severity is Severity.CRITICAL
|
||||
assert len(ups) == 1
|
||||
assert ups[0].detail["device"] == "Pump A"
|
||||
|
||||
|
||||
def test_run_connection_loss_emits_d1_then_reconnect_d2() -> None:
|
||||
"""A failing poll ends the session: DISCONNECTED (D1), then RECONNECTED (D2)."""
|
||||
dying = FakeOpcClient(
|
||||
script=[
|
||||
"good",
|
||||
PollScriptError("could not read from server (HRESULT 0x800706BA: ...)"),
|
||||
]
|
||||
)
|
||||
healthy = FakeOpcClient() # the reconnect lands here
|
||||
collector = _Collector()
|
||||
factory = _FakeFactory([dying, healthy])
|
||||
poller = Poller(factory, _config(), collector)
|
||||
_run_until(poller, lambda: len(collector.of(EventKind.RECONNECTED)) >= 1)
|
||||
|
||||
disconnected = collector.of(EventKind.DISCONNECTED)
|
||||
assert any("HRESULT 0x800706BA" in d.detail.get("reason", "") for d in disconnected)
|
||||
reconnected = collector.of(EventKind.RECONNECTED)
|
||||
assert reconnected and reconnected[0].detail["outage_s"] < 10.0
|
||||
assert factory.created == 2 # one client per session
|
||||
|
||||
|
||||
def test_run_heartbeat_emits_periodically() -> None:
|
||||
# indicator defaults to a plain read -> value 0, which is NOT an ok value
|
||||
# ("true"): script the first poll so the device starts healthy.
|
||||
client = FakeOpcClient(
|
||||
script=[
|
||||
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
|
||||
]
|
||||
)
|
||||
collector = _Collector()
|
||||
poller = Poller(
|
||||
_FakeFactory([client]), _config(heartbeat_s=0.15), collector
|
||||
)
|
||||
_run_until(poller, lambda: len(collector.of(EventKind.HEARTBEAT)) >= 2, timeout_s=10.0)
|
||||
|
||||
heartbeats = collector.of(EventKind.HEARTBEAT)
|
||||
assert len(heartbeats) >= 2
|
||||
detail = heartbeats[0].detail
|
||||
assert detail["tags_total"] == 3 # 2 surveillance + 1 indicator
|
||||
# after the scripted healthy poll the fake falls back to value=0 for the
|
||||
# indicator -> the device is DOWN in later heartbeats. Assert the DOWN
|
||||
# count is reflected honestly rather than expecting fake defaults to be ok.
|
||||
assert detail["devices_down"] in (0, 1)
|
||||
assert detail["reconnects_total"] == 0
|
||||
# a DEVICE_LINK_DOWN must have been reported when it flipped
|
||||
assert collector.of(EventKind.DEVICE_LINK_DOWN) or detail["devices_down"] == 0
|
||||
|
||||
|
||||
def test_run_soft_hang_flags_d7_but_keeps_thread() -> None:
|
||||
"""A slow-but-returning poll raises POLL_HANG (D7) without rebuilding."""
|
||||
|
||||
class SlowClient(FakeOpcClient):
|
||||
def read(self, tags, source=OPCDataSource.DEVICE):
|
||||
time.sleep(0.3) # > timeout_ms 200 below
|
||||
return super().read(tags, source)
|
||||
|
||||
config = _config(poll=PollConfig(interval_s=0.05, timeout_ms=200, hard_hang_s=30.0))
|
||||
client = SlowClient()
|
||||
collector = _Collector()
|
||||
factory = _FakeFactory([client])
|
||||
poller = Poller(factory, config, collector)
|
||||
_run_until(poller, lambda: bool(collector.of(EventKind.POLL_HANG)))
|
||||
|
||||
hangs = collector.of(EventKind.POLL_HANG)
|
||||
assert hangs and hangs[0].detail["rtt_ms"] >= 200
|
||||
assert hangs[0].detail["timeout_ms"] == 200
|
||||
assert factory.created == 1 # thread kept: no rebuild
|
||||
# no abandoned-thread message (that's the hard-hang path)
|
||||
assert all("abandoned" not in h.detail.get("action", "") for h in hangs)
|
||||
|
||||
|
||||
def test_run_hard_hang_abandons_thread_and_rebuilds() -> None:
|
||||
"""Silence > hard_hang_s: the worker is abandoned (POLL_HANG) and rebuilt."""
|
||||
|
||||
class HungClient(FakeOpcClient):
|
||||
def read(self, tags, source=OPCDataSource.DEVICE):
|
||||
time.sleep(10.0) # way past hard_hang_s
|
||||
return super().read(tags, source)
|
||||
|
||||
config = _config(poll=PollConfig(interval_s=0.05, timeout_ms=200, hard_hang_s=0.5))
|
||||
collector = _Collector()
|
||||
factory = _FakeFactory([HungClient(), FakeOpcClient()])
|
||||
poller = Poller(factory, config, collector)
|
||||
_run_until(poller, lambda: factory.created >= 2, timeout_s=10.0)
|
||||
|
||||
hard = [
|
||||
e for e in collector.of(EventKind.POLL_HANG) if "abandoned" in e.detail.get("action", "")
|
||||
]
|
||||
assert hard and hard[0].severity is Severity.CRITICAL
|
||||
assert factory.created == 2 # the hung worker was replaced
|
||||
Reference in New Issue
Block a user