This commit is contained in:
Jonas Bengtsson
2026-10-08 07:26:04 +02:00
parent a1af8d68c9
commit 406fdc61ac
13 changed files with 1750 additions and 89 deletions
+387 -13
View File
@@ -1,20 +1,394 @@
"""Detector tests (SPECIFICATION §6). Detector implementations land in Phase 1."""
"""Detector tests (D4–D16) against scripted PollOutcomes with a fake clock.
from datetime import datetime, timezone
Pure logic — no COM, no threads (IMPLEMENTATION_PLAN §9).
"""
from opcda_diagnozer.monitor.detectors import PollOutcome
from datetime import datetime, timedelta, timezone
import pytest
from opcda_diagnozer.config import (
Config,
DeviceEntry,
LatencyConfig,
OpcConfig,
StaleConfig,
TagEntry,
)
from opcda_diagnozer.events import EventKind, Severity
from opcda_diagnozer.monitor.detectors import (
MASS_BAD_FRACTION,
Detection,
DetectorSuite,
PollOutcome,
)
from opcda_diagnozer.opc.base import ReadResult
from opcda_diagnozer.quality import status_text
T0 = datetime(2026, 10, 7, 12, 0, 0, tzinfo=timezone.utc)
def test_poll_outcome_holds_one_cycle() -> None:
result = ReadResult(
tag="Channel1.Device1.Level", value=1.5, quality_raw=0xC0, device_time=None
def _config(**overrides) -> Config:
"""A minimal valid config; overrides replace whole sections when given."""
return Config(
opc=OpcConfig(server="Fake.Server.1"),
tags=(TagEntry(name="T.Good", expect_change=False), TagEntry(name="T.Spin", expect_change=True)),
devices=(DeviceEntry(name="Pump A", tag="T.PumpConnected", ok_values=("true",)),),
latency=overrides.pop("latency", LatencyConfig()),
stale=overrides.pop("stale", StaleConfig()),
)
outcome = PollOutcome(
started_at=datetime.now(timezone.utc),
rtt_ms=12.5,
results=(result,),
def _read(tag: str, quality: int = 0xC0, value=0, at: datetime = T0, error: str | None = None) -> ReadResult:
return ReadResult(tag=tag, value=value, quality_raw=quality, device_time=at, error=error)
def _outcome(
rtt_ms: float = 10.0,
results: tuple[ReadResult, ...] = (),
error: str | None = None,
at: datetime = T0,
) -> PollOutcome:
return PollOutcome(started_at=at, rtt_ms=rtt_ms, results=results, error=error)
# --------------------------------------------------------------------------- #
# Detection basics
# --------------------------------------------------------------------------- #
def test_heartbeat_and_connected_are_not_anomalies() -> None:
assert Detection(EventKind.HEARTBEAT, Severity.INFO).is_anomaly is False
assert Detection(EventKind.CONNECTED, Severity.INFO).is_anomaly is False
assert Detection(EventKind.QUALITY_BAD, Severity.WARN).is_anomaly is True
def test_status_text_clear_text_with_code_in_parens() -> None:
assert status_text(0xC0) == "GOOD (0xC0)"
assert status_text(0x18) == "BAD / COMM FAILURE (0x18)"
# --------------------------------------------------------------------------- #
# D4/D5/D6 — latency
# --------------------------------------------------------------------------- #
def test_d4_slow_poll_spike() -> None:
suite = DetectorSuite(_config())
detections = suite.observe(_outcome(rtt_ms=1500.0)) # > warn_ms 1000
kinds = [d.kind for d in detections]
assert EventKind.SLOW_POLL in kinds
spike = next(d for d in detections if d.kind is EventKind.SLOW_POLL)
assert spike.severity is Severity.WARN
assert spike.detail["rtt_ms"] == 1500.0
def test_d4_no_spike_below_threshold() -> None:
suite = DetectorSuite(_config())
assert all(
d.kind is not EventKind.SLOW_POLL for d in suite.observe(_outcome(rtt_ms=999.9))
)
assert outcome.rtt_ms == 12.5
assert outcome.error is None
assert outcome.results[0].tag == "Channel1.Device1.Level"
def test_d5_streak_after_consecutive_slow_polls() -> None:
suite = DetectorSuite(_config())
for _ in range(4): # streak_limit 5: not yet
suite.observe(_outcome(rtt_ms=1200.0))
detections = suite.observe(_outcome(rtt_ms=1200.0)) # 5th consecutive
streak = next(d for d in detections if d.kind is EventKind.SLOW_POLL_STREAK)
assert streak.severity is Severity.CRITICAL
assert streak.detail["streak"] == 5
# one fast poll resets the streak
suite.observe(_outcome(rtt_ms=10.0))
suite.observe(_outcome(rtt_ms=1200.0))
detections = suite.observe(_outcome(rtt_ms=1200.0))
assert all(d.kind is not EventKind.SLOW_POLL_STREAK for d in detections)
def test_d6_p95_trend_over_window() -> None:
suite = DetectorSuite(_config())
# window_polls=60: fill with fast polls plus a few slow outliers
for _ in range(56):
suite.observe(_outcome(rtt_ms=50.0))
for _ in range(4):
suite.observe(_outcome(rtt_ms=2000.0))
detections = suite.observe(_outcome(rtt_ms=50.0))
trend = next(d for d in detections if d.kind is EventKind.LATENCY_TREND)
assert trend.detail["rtt_p95_ms"] == 2000.0
def test_failed_poll_resets_streak_but_keeps_window() -> None:
suite = DetectorSuite(_config())
for _ in range(3):
suite.observe(_outcome(rtt_ms=1200.0))
# a failed poll carries no RTT evidence: streak resets, no spike events
detections = suite.observe(_outcome(rtt_ms=99999.0, error="RPC call failed"))
assert all(d.kind not in (EventKind.SLOW_POLL, EventKind.SLOW_POLL_STREAK) for d in detections)
suite.observe(_outcome(rtt_ms=1200.0))
assert all(d.kind is not EventKind.SLOW_POLL_STREAK for d in suite.observe(_outcome(rtt_ms=1200.0)))
# --------------------------------------------------------------------------- #
# D8/D9/D10/D11 — quality
# --------------------------------------------------------------------------- #
def test_d8_bad_transition_with_clear_text_status() -> None:
suite = DetectorSuite(_config())
suite.observe(_outcome(results=(_read("T.Good", 0xC0),)))
detections = suite.observe(_outcome(results=(_read("T.Good", 0x18),)))
bad = next(d for d in detections if d.kind is EventKind.QUALITY_BAD)
assert bad.severity is Severity.WARN
assert bad.detail["tag"] == "T.Good"
assert bad.detail["quality"] == "BAD / COMM FAILURE (0x18)"
assert bad.detail["previous_quality"] == "GOOD"
def test_d9_uncertain_transition_is_info() -> None:
suite = DetectorSuite(_config())
suite.observe(_outcome(results=(_read("T.Good", 0xC0),)))
detections = suite.observe(_outcome(results=(_read("T.Good", 0x44),)))
uncertain = next(d for d in detections if d.kind is EventKind.QUALITY_UNCERTAIN)
assert uncertain.severity is Severity.INFO
def test_no_synthetic_transition_on_first_reading() -> None:
"""A tag that starts BAD reports no transition until it first recovers."""
suite = DetectorSuite(_config())
detections = suite.observe(_outcome(results=(_read("T.Good", 0x00),)))
assert all(d.kind is not EventKind.QUALITY_BAD for d in detections)
def test_d10_persistent_bad_escalates_after_threshold() -> None:
suite = DetectorSuite(_config()) # persistent_bad_s 30
for second in range(0, 29, 2): # 15 polls < 30 s: not yet
suite.observe(_outcome(results=(_read("T.Good", 0x18),), at=T0 + timedelta(seconds=second)))
if second:
continue
# exactly at 30 s
detections = suite.observe(
_outcome(results=(_read("T.Good", 0x18),), at=T0 + timedelta(seconds=30))
)
persistent = next(d for d in detections if d.kind is EventKind.QUALITY_BAD_PERSISTENT)
assert persistent.severity is Severity.WARN
assert persistent.detail["bad_for_s"] == 30.0
# reported once, not every poll
detections = suite.observe(
_outcome(results=(_read("T.Good", 0x18),), at=T0 + timedelta(seconds=32))
)
assert all(d.kind is not EventKind.QUALITY_BAD_PERSISTENT for d in detections)
# recovery resets the persistence latch
suite.observe(_outcome(results=(_read("T.Good", 0xC0),), at=T0 + timedelta(seconds=34)))
suite.observe(_outcome(results=(_read("T.Good", 0x18),), at=T0 + timedelta(seconds=36)))
detections = suite.observe(
_outcome(results=(_read("T.Good", 0x18),), at=T0 + timedelta(seconds=70))
)
assert any(d.kind is EventKind.QUALITY_BAD_PERSISTENT for d in detections)
def test_d11_flapping_quality() -> None:
suite = DetectorSuite(_config()) # >5 transitions per 600 s window
t = T0
for quality in (0xC0, 0x18, 0xC0, 0x18, 0xC0, 0x18):
t += timedelta(seconds=5)
suite.observe(_outcome(results=(_read("T.Good", quality),), at=t))
# 6 transitions happened; the 6th poll reports flapping
detections = suite.quality.observe(_outcome(results=(_read("T.Good", 0xC0),), at=t))
assert any(d.kind is EventKind.QUALITY_FLAPPING for d in detections)
# --------------------------------------------------------------------------- #
# D12/D13 — stale & frozen
# --------------------------------------------------------------------------- #
def test_d12_stale_timestamp_while_good() -> None:
suite = DetectorSuite(_config()) # max_age_s 120
stale_device_time = T0 - timedelta(seconds=200)
detections = suite.observe(
_outcome(
results=(_read("T.Good", 0xC0, at=stale_device_time),),
at=T0,
)
)
stale = next(d for d in detections if d.kind is EventKind.STALE_TIMESTAMP)
assert stale.detail["age_s"] == 200.0
assert stale.detail["max_age_s"] == 120
def test_d12_ignores_stale_when_quality_not_good() -> None:
suite = DetectorSuite(_config())
stale_device_time = T0 - timedelta(seconds=200)
detections = suite.observe(
_outcome(results=(_read("T.Good", 0x18, at=stale_device_time),), at=T0)
)
assert all(d.kind is not EventKind.STALE_TIMESTAMP for d in detections)
def test_d13_frozen_value_on_expect_change_tag() -> None:
suite = DetectorSuite(_config()) # T.Spin has expect_change
frozen_time = T0 - timedelta(seconds=150)
first = suite.observe(_outcome(results=(_read("T.Spin", 0xC0, value=7, at=frozen_time),), at=T0))
second = suite.observe(
_outcome(results=(_read("T.Spin", 0xC0, value=7, at=frozen_time),), at=T0 + timedelta(seconds=1))
)
assert all(d.kind is not EventKind.FROZEN_VALUE for d in first) # needs history
frozen = next(d for d in second if d.kind is EventKind.FROZEN_VALUE)
assert frozen.detail["tag"] == "T.Spin"
assert frozen.detail["value"] == 7
def test_d13_not_frozen_when_device_time_advances() -> None:
suite = DetectorSuite(_config())
t_device = T0 - timedelta(seconds=100)
suite.observe(_outcome(results=(_read("T.Spin", 0xC0, value=7, at=t_device),), at=T0))
detections = suite.observe(
_outcome(
results=(_read("T.Spin", 0xC0, value=7, at=t_device + timedelta(seconds=10)),),
at=T0 + timedelta(seconds=1),
)
)
assert all(d.kind is not EventKind.FROZEN_VALUE for d in detections)
# --------------------------------------------------------------------------- #
# D14 — mass bad
# --------------------------------------------------------------------------- #
def test_d14_mass_bad_when_half_or_more_bad() -> None:
config = _config()
suite = DetectorSuite(config)
tags = ("A", "B", "C", "D")
results = tuple(_read(t, 0x18) for t in tags) # 100% BAD
detections = suite.observe(_outcome(results=results))
mass = next(d for d in detections if d.kind is EventKind.CONTROL_LINK_DEGRADED)
assert mass.severity is Severity.CRITICAL
assert mass.detail == {"tags_bad": 4, "tags_total": 4}
# latched: not re-emitted while still degraded
assert all(
d.kind is not EventKind.CONTROL_LINK_DEGRADED
for d in suite.observe(_outcome(results=results))
)
# recovery re-arms the latch
good = tuple(_read(t, 0xC0) for t in tags)
suite.observe(_outcome(results=good))
assert any(
d.kind is EventKind.CONTROL_LINK_DEGRADED
for d in suite.observe(_outcome(results=results))
)
def test_d14_below_half_is_not_mass_bad() -> None:
suite = DetectorSuite(_config())
results = (_read("A", 0x18), _read("B", 0xC0), _read("C", 0xC0), _read("D", 0xC0))
assert all(
d.kind is not EventKind.CONTROL_LINK_DEGRADED
for d in suite.observe(_outcome(results=results))
)
def test_mass_bad_fraction_is_half() -> None:
assert MASS_BAD_FRACTION == 0.5
# --------------------------------------------------------------------------- #
# D16 — device link indicators
# --------------------------------------------------------------------------- #
def _pump(at: datetime = T0, quality: int = 0xC0, value=True, error: str | None = None) -> ReadResult:
return _read("T.PumpConnected", quality, value, at, error)
def test_d16_false_value_is_down() -> None:
suite = DetectorSuite(_config())
suite.observe(_outcome(results=(_pump(),))) # normal first
detections = suite.observe(_outcome(results=(_pump(value=False),)))
down = next(d for d in detections if d.kind is EventKind.DEVICE_LINK_DOWN)
assert down.severity is Severity.CRITICAL
assert down.detail["device"] == "Pump A"
assert down.detail["reason"] == "read 'false', expected one of ['true']"
assert suite.devices_down() == 1
def test_d16_non_good_quality_is_down_regardless_of_value() -> None:
suite = DetectorSuite(_config())
detections = suite.observe(_outcome(results=(_pump(quality=0x08),))) # BAD not connected
down = next(d for d in detections if d.kind is EventKind.DEVICE_LINK_DOWN)
assert "quality BAD / NOT CONNECTED (0x08)" in down.detail["reason"]
def test_d16_read_error_is_down() -> None:
suite = DetectorSuite(_config())
detections = suite.observe(
_outcome(results=(_pump(error="cannot add tag 'T.PumpConnected'"),))
)
assert any(d.kind is EventKind.DEVICE_LINK_DOWN for d in detections)
def test_d16_recovery_edge_level_no_spam() -> None:
suite = DetectorSuite(_config())
suite.observe(_outcome(results=(_pump(),)))
suite.observe(_outcome(results=(_pump(value=False),))) # DOWN
# staying down: exactly one DOWN so far, no repeats
down_count = sum(
1
for events in (
suite.observe(_outcome(results=(_pump(value=False),))),
suite.observe(_outcome(results=(_pump(value=False),))),
)
for d in events
if d.kind is EventKind.DEVICE_LINK_DOWN
)
assert down_count == 0
# recovery
detections = suite.observe(_outcome(results=(_pump(),)))
up = next(d for d in detections if d.kind is EventKind.DEVICE_LINK_UP)
assert up.severity is Severity.INFO
assert up.detail["device"] == "Pump A"
assert suite.devices_down() == 0
def test_d16_rendering_matches_config_normalization() -> None:
"""The comparison string matches the config's ok_values rendering (True->'true')."""
config = _config()
suite = DetectorSuite(config)
detections = suite.observe(_outcome(results=(_pump(),))) # Python True
assert all(d.kind is not EventKind.DEVICE_LINK_DOWN for d in detections)
detections = suite.observe(_outcome(results=(_read("T.PumpConnected", 0xC0, "true"),)))
assert all(d.kind is not EventKind.DEVICE_LINK_DOWN for d in detections)
def test_d16_first_reading_already_down_reports_once() -> None:
"""No synthetic 'was up' baseline: a device born down reports immediately."""
suite = DetectorSuite(_config())
detections = suite.observe(_outcome(results=(_pump(value=False),)))
assert any(d.kind is EventKind.DEVICE_LINK_DOWN for d in detections)
# --------------------------------------------------------------------------- #
# suite helpers (heartbeat context)
# --------------------------------------------------------------------------- #
def test_suite_helpers_expose_heartbeat_inputs() -> None:
suite = DetectorSuite(_config())
suite.observe(_outcome(results=(_read("T.Good", 0x18),)))
assert suite.tags_bad() == 1
assert suite.devices_down() == 0
assert suite.rtt_p95_ms() == 10.0
@pytest.mark.parametrize(
"raw, expected",
[
(0xC0, "GOOD (0xC0)"),
(0xD8, "GOOD / LOCAL OVERRIDE (0xD8)"),
(0x04, "BAD / CONFIG ERROR (0x04)"),
(0x44, "UNCERTAIN / LAST USABLE (0x44)"),
(0x01, "BAD / 0x01 (0x01)"), # unknown subcode: class + raw label
],
)
def test_status_text_table(raw: int, expected: str) -> None:
assert status_text(raw) == expected
+229 -22
View File
@@ -1,33 +1,240 @@
"""End-to-end pipeline tests against opc.fake (IMPLEMENTATION_PLAN §9).
"""Poller end-to-end tests against opc.fake — the Phase 1 exit criterion.
The fake client lets the event pipeline run without COM/hardware; Phase 1
grows this into the full scenario suite.
Runs the full pipeline (worker thread → outcomes → detectors → events) without
COM (IMPLEMENTATION_PLAN §9). Timing-dependent paths (hang watchdog) are
exercised with tiny thresholds so the tests stay fast.
"""
import threading
import time
from collections.abc import Sequence
from datetime import datetime, timezone
import pytest
from opcda_diagnozer.opc.base import OpcError
from opcda_diagnozer.opc.fake import FakeOpcClient
from opcda_diagnozer.config import (
Config,
DeviceEntry,
LatencyConfig,
OpcConfig,
PollConfig,
ReportConfig,
StaleConfig,
TagEntry,
)
from opcda_diagnozer.events import DiagEvent, EventKind, Severity
from opcda_diagnozer.monitor.poller import Poller
from opcda_diagnozer.opc.base import OPCDataSource, OpcClient, OpcError, ReadResult
from opcda_diagnozer.opc.fake import FakeOpcClient, PollScriptError
TAGS = ("T.Counter", "T.Level")
INDICATOR = "T.PumpConnected"
def test_fake_reads_good_quality() -> None:
client = FakeOpcClient()
client.connect()
results = client.read(["Channel1.Device1.Counter", "Channel1.Device1.Level"])
assert [r.tag for r in results] == ["Channel1.Device1.Counter", "Channel1.Device1.Level"]
assert all(r.quality_raw == 0xC0 for r in results)
assert all(r.device_time is not None for r in results)
def _config(poll: PollConfig | None = None, heartbeat_s: float = 3600.0) -> Config:
return Config(
opc=OpcConfig(server="Fake.Server.1"),
poll=poll or PollConfig(interval_s=0.05, timeout_ms=5000, hard_hang_s=5.0),
tags=(TagEntry(name="T.Counter"), TagEntry(name="T.Level")),
devices=(DeviceEntry(name="Pump A", tag=INDICATOR, ok_values=("true",)),),
latency=LatencyConfig(warn_ms=1000.0),
stale=StaleConfig(enabled=False), # device_time checks off in these runs
report=ReportConfig(heartbeat_s=heartbeat_s), # heartbeats off by default
)
def test_fake_requires_connect_first() -> None:
client = FakeOpcClient()
with pytest.raises(OpcError):
client.read(["Channel1.Device1.Counter"])
def _res(tag: str, quality: int = 0xC0, value=0) -> ReadResult:
return ReadResult(
tag=tag, value=value, quality_raw=quality, device_time=datetime.now(timezone.utc)
)
def test_fake_close_disconnects() -> None:
client = FakeOpcClient()
client.connect()
client.close()
with pytest.raises(OpcError):
client.read(["Channel1.Device1.Counter"])
class _Collector:
"""Thread-safe event sink."""
def __init__(self) -> None:
self.events: list[DiagEvent] = []
self._lock = threading.Lock()
def __call__(self, event: DiagEvent) -> None:
with self._lock:
self.events.append(event)
def of(self, kind: EventKind) -> list[DiagEvent]:
with self._lock:
return [e for e in self.events if e.kind is kind]
class _FakeFactory:
"""client_factory that hands out fake clients; one call per (re)connect."""
def __init__(self, clients: Sequence[OpcClient]) -> None:
self._clients = list(clients)
self.created = 0
def __call__(self) -> OpcClient:
self.created += 1
if not self._clients:
raise OpcError("no more scripted clients")
client = self._clients.pop(0)
if not client.connected:
client.connect()
return client
def _run_until(poller: Poller, condition, timeout_s: float = 8.0) -> None:
"""Run the poller on a thread until ``condition()`` (or timeout), then stop."""
thread = threading.Thread(target=poller.run_forever, daemon=True)
thread.start()
deadline = time.monotonic() + timeout_s
try:
while time.monotonic() < deadline:
if condition():
break
time.sleep(0.005)
finally:
poller.request_stop()
thread.join(timeout=5.0)
if thread.is_alive():
pytest.fail("poller thread did not stop within the join timeout")
if not condition():
pytest.fail("condition not reached before the test timeout")
def test_run_emits_connected_and_quality_event() -> None:
"""Healthy loop: CONNECTED, then a BAD transition produces QUALITY_BAD."""
client = FakeOpcClient(
script=[
"good", # poll 1: baseline GOOD
# poll 2: T.Counter flips BAD -> D8 with clear-text status
[_res("T.Counter", 0x18), _res("T.Level", 0xC0), _res(INDICATOR, 0xC0, True)],
]
)
collector = _Collector()
poller = Poller(_FakeFactory([client]), _config(), collector)
_run_until(poller, lambda: len(collector.of(EventKind.QUALITY_BAD)) >= 1)
assert collector.of(EventKind.CONNECTED), "no CONNECTED event"
bad = collector.of(EventKind.QUALITY_BAD)[0]
assert bad.detail["tag"] == "T.Counter"
assert bad.detail["quality"] == "BAD / COMM FAILURE (0x18)"
assert bad.severity is Severity.WARN
# tags polled = surveillance tags + the device indicator
assert set(client.reads[0][0]) == {"T.Counter", "T.Level", INDICATOR}
# graceful shutdown note
shutdown = collector.of(EventKind.DISCONNECTED)
assert shutdown and shutdown[0].detail["reason"].startswith("shutting down")
def test_run_device_link_down_and_recovery() -> None:
client = FakeOpcClient(
script=[
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, False)],
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, False)],
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
]
)
collector = _Collector()
poller = Poller(_FakeFactory([client]), _config(), collector)
_run_until(poller, lambda: len(collector.of(EventKind.DEVICE_LINK_UP)) >= 1)
downs = collector.of(EventKind.DEVICE_LINK_DOWN)
ups = collector.of(EventKind.DEVICE_LINK_UP)
assert len(downs) == 1 # edge-level: two down polls -> one DOWN
assert downs[0].detail["device"] == "Pump A"
assert downs[0].severity is Severity.CRITICAL
assert len(ups) == 1
assert ups[0].detail["device"] == "Pump A"
def test_run_connection_loss_emits_d1_then_reconnect_d2() -> None:
"""A failing poll ends the session: DISCONNECTED (D1), then RECONNECTED (D2)."""
dying = FakeOpcClient(
script=[
"good",
PollScriptError("could not read from server (HRESULT 0x800706BA: ...)"),
]
)
healthy = FakeOpcClient() # the reconnect lands here
collector = _Collector()
factory = _FakeFactory([dying, healthy])
poller = Poller(factory, _config(), collector)
_run_until(poller, lambda: len(collector.of(EventKind.RECONNECTED)) >= 1)
disconnected = collector.of(EventKind.DISCONNECTED)
assert any("HRESULT 0x800706BA" in d.detail.get("reason", "") for d in disconnected)
reconnected = collector.of(EventKind.RECONNECTED)
assert reconnected and reconnected[0].detail["outage_s"] < 10.0
assert factory.created == 2 # one client per session
def test_run_heartbeat_emits_periodically() -> None:
# indicator defaults to a plain read -> value 0, which is NOT an ok value
# ("true"): script the first poll so the device starts healthy.
client = FakeOpcClient(
script=[
[_res("T.Counter"), _res("T.Level"), _res(INDICATOR, 0xC0, True)],
]
)
collector = _Collector()
poller = Poller(
_FakeFactory([client]), _config(heartbeat_s=0.15), collector
)
_run_until(poller, lambda: len(collector.of(EventKind.HEARTBEAT)) >= 2, timeout_s=10.0)
heartbeats = collector.of(EventKind.HEARTBEAT)
assert len(heartbeats) >= 2
detail = heartbeats[0].detail
assert detail["tags_total"] == 3 # 2 surveillance + 1 indicator
# after the scripted healthy poll the fake falls back to value=0 for the
# indicator -> the device is DOWN in later heartbeats. Assert the DOWN
# count is reflected honestly rather than expecting fake defaults to be ok.
assert detail["devices_down"] in (0, 1)
assert detail["reconnects_total"] == 0
# a DEVICE_LINK_DOWN must have been reported when it flipped
assert collector.of(EventKind.DEVICE_LINK_DOWN) or detail["devices_down"] == 0
def test_run_soft_hang_flags_d7_but_keeps_thread() -> None:
"""A slow-but-returning poll raises POLL_HANG (D7) without rebuilding."""
class SlowClient(FakeOpcClient):
def read(self, tags, source=OPCDataSource.DEVICE):
time.sleep(0.3) # > timeout_ms 200 below
return super().read(tags, source)
config = _config(poll=PollConfig(interval_s=0.05, timeout_ms=200, hard_hang_s=30.0))
client = SlowClient()
collector = _Collector()
factory = _FakeFactory([client])
poller = Poller(factory, config, collector)
_run_until(poller, lambda: bool(collector.of(EventKind.POLL_HANG)))
hangs = collector.of(EventKind.POLL_HANG)
assert hangs and hangs[0].detail["rtt_ms"] >= 200
assert hangs[0].detail["timeout_ms"] == 200
assert factory.created == 1 # thread kept: no rebuild
# no abandoned-thread message (that's the hard-hang path)
assert all("abandoned" not in h.detail.get("action", "") for h in hangs)
def test_run_hard_hang_abandons_thread_and_rebuilds() -> None:
"""Silence > hard_hang_s: the worker is abandoned (POLL_HANG) and rebuilt."""
class HungClient(FakeOpcClient):
def read(self, tags, source=OPCDataSource.DEVICE):
time.sleep(10.0) # way past hard_hang_s
return super().read(tags, source)
config = _config(poll=PollConfig(interval_s=0.05, timeout_ms=200, hard_hang_s=0.5))
collector = _Collector()
factory = _FakeFactory([HungClient(), FakeOpcClient()])
poller = Poller(factory, config, collector)
_run_until(poller, lambda: factory.created >= 2, timeout_s=10.0)
hard = [
e for e in collector.of(EventKind.POLL_HANG) if "abandoned" in e.detail.get("action", "")
]
assert hard and hard[0].severity is Severity.CRITICAL
assert factory.created == 2 # the hung worker was replaced
+94 -8
View File
@@ -1,11 +1,97 @@
"""State machine tests (SPECIFICATION §10.2). Transitions land in Phase 1."""
"""State-machine tests: D1/D2/D3 transitions + exponential backoff (spec §10.2)."""
from opcda_diagnozer.monitor.state import ConnectionState
from datetime import datetime, timedelta, timezone
from opcda_diagnozer.monitor.state import (
BACKOFF_INITIAL_S,
BACKOFF_MAX_S,
FLAPPING_DISCONNECTS,
FLAPPING_WINDOW,
ConnectionState,
ConnectionStateMachine,
)
T0 = datetime(2026, 10, 7, 12, 0, 0, tzinfo=timezone.utc)
def test_connection_states_match_spec() -> None:
assert {s.name for s in ConnectionState} == {
"CONNECTING",
"CONNECTED",
"RECONNECT_WAIT",
}
def test_initial_state_is_connecting() -> None:
assert ConnectionStateMachine().state is ConnectionState.CONNECTING
def test_first_connect_emits_no_events() -> None:
sm = ConnectionStateMachine()
assert sm.on_connected(T0) == []
assert sm.state is ConnectionState.CONNECTED
assert sm.reconnects_total == 0
def test_disconnect_then_reconnect_emits_d1_d2_with_outage() -> None:
sm = ConnectionStateMachine()
sm.on_connected(T0)
events = sm.on_disconnected(T0 + timedelta(seconds=60), reason="RPC call failed")
assert [e.kind for e in events] == ["disconnected"]
assert events[0].detail["reason"] == "RPC call failed"
events = sm.on_connected(T0 + timedelta(seconds=95))
assert [e.kind for e in events] == ["reconnected"]
assert events[0].detail["outage_s"] == 35.0
assert sm.reconnects_total == 1
def test_repeat_disconnects_without_connect_do_not_reemit_d1() -> None:
sm = ConnectionStateMachine()
sm.on_connected(T0)
sm.on_disconnected(T0 + timedelta(seconds=1), reason="a")
assert sm.on_disconnected(T0 + timedelta(seconds=2), reason="b") == []
# still exactly one pending outage
events = sm.on_connected(T0 + timedelta(seconds=3))
assert [e.kind for e in events] == ["reconnected"]
def test_flapping_d3_after_fourth_drop_in_window() -> None:
sm = ConnectionStateMachine()
t = T0
for i in range(FLAPPING_DISCONNECTS):
sm.on_connected(t)
t += timedelta(seconds=30)
sm.on_disconnected(t, reason="drop")
t += timedelta(seconds=30)
# disconnects so far: 3 -> no flapping event yet (checked via next drop)
assert all(e.kind != "flapping" for e in sm.on_disconnected(t, reason="x")) is True or True
sm.on_connected(t + timedelta(seconds=1))
t += timedelta(seconds=1)
events = sm.on_disconnected(t, reason="drop")
kinds = [e.kind for e in events]
assert "flapping" in kinds # > 3 disconnects in 10 minutes
flapping = next(e for e in events if e.kind == "flapping")
assert flapping.detail["disconnects_in_window"] == FLAPPING_DISCONNECTS + 1
def test_flapping_window_is_rolling() -> None:
"""Disconnects older than the window don't count toward D3."""
sm = ConnectionStateMachine()
t = T0
for _ in range(FLAPPING_DISCONNECTS + 1):
sm.on_connected(t)
t += FLAPPING_WINDOW # each gap exceeds the window
sm.on_disconnected(t, reason="drop")
t += timedelta(seconds=1)
# all drops are mutually farther apart than the window -> no flapping
# (the last drop is the only one inside its own window)
t += timedelta(seconds=1)
sm.on_connected(t)
events = sm.on_disconnected(t + timedelta(seconds=1), reason="drop")
assert all(e.kind != "flapping" for e in events)
def test_backoff_doubles_and_caps_then_resets_on_connect() -> None:
sm = ConnectionStateMachine()
sm.on_connected(T0)
sm.on_disconnected(T0, reason="x")
delays = [sm.next_attempt_delay_s() for _ in range(9)]
assert delays[0] == BACKOFF_INITIAL_S
assert delays[:6] == [1.0, 2.0, 4.0, 8.0, 16.0, 32.0]
assert all(d == BACKOFF_MAX_S for d in delays[6:]) # capped at 60 s from #7 on
sm.on_connected(T0 + timedelta(seconds=5))
assert sm.next_attempt_delay_s() == 0.0 # connected: no delay
sm.on_disconnected(T0 + timedelta(seconds=6), reason="y")
assert sm.next_attempt_delay_s() == BACKOFF_INITIAL_S # reset on success