🚧 Add topic explorer
This commit is contained in:
Jonas Bengtsson
@@ -0,0 +1,318 @@
|
||||
"""Topic explorer sessions (spec sections 35, 36).
|
||||
|
||||
One MQTT v5 client per active browser stream. Sessions live only while
|
||||
the Server-Sent-Events connection is open; the stream generator's
|
||||
finally-clause stops the session, which disconnects the client — no
|
||||
lingering subscriptions, no manual cleanup by the operator.
|
||||
|
||||
Why MQTT v5: the explorer must surface MQTT 5 publish properties (spec
|
||||
section 35), which only arrive when the client itself speaks v5. The
|
||||
dynsec management channel deliberately stays on its own connection.
|
||||
|
||||
Payloads are base64 with a looks-like-UTF-8 flag: messages can be binary,
|
||||
and the UI offers text/hex views accordingly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import paho.mqtt.client as mqtt
|
||||
from paho.mqtt.enums import CallbackAPIVersion
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BUFFER_SIZE = 1000 # ring buffer per session
|
||||
_GRACE_SECONDS = 30.0 # page reload / SSE reconnect window
|
||||
|
||||
|
||||
def _payload_info(payload: bytes) -> dict[str, Any]:
|
||||
"""base64 + heuristic UTF-8 decoding for the UI."""
|
||||
info: dict[str, Any] = {"payload_b64": base64.b64encode(payload).decode()}
|
||||
try:
|
||||
info["payload_text"] = payload.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
info["payload_text"] = None # binary — UI shows hex
|
||||
return info
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Message:
|
||||
index: int
|
||||
topic: str
|
||||
qos: int
|
||||
retain: bool
|
||||
received_at: float
|
||||
payload_b64: str
|
||||
payload_text: str | None
|
||||
properties: dict[str, Any] | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"index": self.index,
|
||||
"topic": self.topic,
|
||||
"qos": self.qos,
|
||||
"retain": self.retain,
|
||||
"received_at": self.received_at,
|
||||
"payload_b64": self.payload_b64,
|
||||
"payload_text": self.payload_text,
|
||||
"properties": self.properties,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExplorerSession:
|
||||
id: str
|
||||
username: str
|
||||
topic_filter: str
|
||||
client: mqtt.Client = field(repr=False)
|
||||
buffer: deque[_Message] = field(default_factory=lambda: deque(maxlen=_BUFFER_SIZE))
|
||||
_next_index: int = 0
|
||||
_cond: threading.Condition = field(default_factory=threading.Condition)
|
||||
_subscribed: threading.Event = field(default_factory=threading.Event)
|
||||
created_at: float = field(default_factory=time.time)
|
||||
|
||||
def _on_connect(self, client, userdata, flags, reason_code, properties) -> None:
|
||||
if reason_code == 0:
|
||||
client.subscribe(self.topic_filter, qos=0)
|
||||
else:
|
||||
logger.error("explorer session %s connect failed: %s", self.id, reason_code)
|
||||
|
||||
def _on_subscribe(self, client, userdata, mid, reason_codes, properties) -> None:
|
||||
# SUBACK received: the subscription is now live on the broker.
|
||||
self._subscribed.set()
|
||||
|
||||
def _on_message(self, client, userdata, msg) -> None:
|
||||
# MQTT v5: surface publish properties (content type, response
|
||||
# topic, user properties, ...). paho exposes them as msg.properties.
|
||||
props: dict[str, Any] | None = None
|
||||
raw_props = getattr(msg, "properties", None)
|
||||
if raw_props is not None:
|
||||
try:
|
||||
props = {}
|
||||
if getattr(raw_props, "ContentType", None):
|
||||
props["content_type"] = raw_props.ContentType
|
||||
if getattr(raw_props, "ResponseTopic", None):
|
||||
props["response_topic"] = raw_props.ResponseTopic
|
||||
if getattr(raw_props, "CorrelationData", None):
|
||||
props["correlation_data"] = raw_props.CorrelationData.decode(
|
||||
"utf-8", errors="replace"
|
||||
)
|
||||
if getattr(raw_props, "UserProperty", None):
|
||||
props["user_properties"] = [
|
||||
[k, v] for k, v in raw_props.UserProperty
|
||||
]
|
||||
except Exception: # noqa: BLE001 - properties must never break capture
|
||||
props = None
|
||||
info = _payload_info(msg.payload)
|
||||
with self._cond:
|
||||
message = _Message(
|
||||
index=self._next_index,
|
||||
topic=msg.topic,
|
||||
qos=msg.qos,
|
||||
retain=bool(getattr(msg, "retain", False)),
|
||||
received_at=time.time(),
|
||||
payload_b64=info["payload_b64"],
|
||||
payload_text=info["payload_text"],
|
||||
properties=props,
|
||||
)
|
||||
self._next_index += 1
|
||||
self.buffer.append(message)
|
||||
self._cond.notify_all()
|
||||
|
||||
|
||||
class ExplorerSessions:
|
||||
"""Thread-safe registry of live explorer sessions."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._sessions: dict[str, ExplorerSession] = {}
|
||||
|
||||
def start(self, username: str, topic_filter: str) -> ExplorerSession:
|
||||
settings = get_settings()
|
||||
session_id = uuid.uuid4().hex
|
||||
client = mqtt.Client(
|
||||
callback_api_version=CallbackAPIVersion.VERSION2,
|
||||
client_id=f"mqtt-explorer-{session_id[:8]}",
|
||||
protocol=mqtt.MQTTv5,
|
||||
)
|
||||
client.username_pw_set(
|
||||
settings.mosquitto_dynsec_username, settings.mosquitto_dynsec_password
|
||||
)
|
||||
session = ExplorerSession(
|
||||
id=session_id,
|
||||
username=username,
|
||||
topic_filter=topic_filter,
|
||||
client=client,
|
||||
)
|
||||
client.on_connect = session._on_connect
|
||||
client.on_subscribe = session._on_subscribe
|
||||
client.on_message = session._on_message
|
||||
client.connect(
|
||||
settings.mosquitto_host, settings.mosquitto_mgmt_port, keepalive=30
|
||||
)
|
||||
client.loop_start()
|
||||
# Block until the broker acknowledges the subscription: without
|
||||
# this, a message published immediately after subscribe returns
|
||||
# can be missed (SUBACK arrives asynchronously on the network
|
||||
# loop). 5 s covers a cold broker start; on timeout the session
|
||||
# still proceeds — delivery may just start slightly later.
|
||||
if not session._subscribed.wait(timeout=5.0):
|
||||
logger.warning(
|
||||
"explorer session %s: SUBACK not received within 5 s",
|
||||
session_id,
|
||||
)
|
||||
with self._lock:
|
||||
self._sessions[session_id] = session
|
||||
return session
|
||||
|
||||
def get(self, session_id: str) -> ExplorerSession:
|
||||
with self._lock:
|
||||
session = self._sessions.get(session_id)
|
||||
if session is None:
|
||||
raise KeyError(session_id)
|
||||
return session
|
||||
|
||||
def stop(self, session_id: str) -> None:
|
||||
with self._lock:
|
||||
session = self._sessions.pop(session_id, None)
|
||||
if session is None:
|
||||
return
|
||||
try:
|
||||
session.client.disconnect()
|
||||
except OSError:
|
||||
pass
|
||||
session.client.loop_stop()
|
||||
|
||||
def sweep(self) -> None:
|
||||
"""Drop sessions whose buffer has gone silent past the grace
|
||||
window (browser tab closed without the SSE finally firing)."""
|
||||
cutoff = time.time() - _GRACE_SECONDS
|
||||
with self._lock:
|
||||
stale = [
|
||||
sid
|
||||
for sid, s in self._sessions.items()
|
||||
if s.buffer and s.buffer[-1].received_at < cutoff
|
||||
and time.time() - s.created_at > _GRACE_SECONDS
|
||||
]
|
||||
for sid in stale:
|
||||
self.stop(sid)
|
||||
|
||||
def stop_all(self) -> None:
|
||||
with self._lock:
|
||||
ids = list(self._sessions)
|
||||
for sid in ids:
|
||||
self.stop(sid)
|
||||
|
||||
|
||||
sessions = ExplorerSessions()
|
||||
|
||||
|
||||
def messages_since(session_id: str, index: int) -> list[dict[str, Any]]:
|
||||
"""All buffered messages with index > the given one."""
|
||||
session = sessions.get(session_id)
|
||||
with session._cond:
|
||||
return [m.as_dict() for m in session.buffer if m.index > index]
|
||||
|
||||
|
||||
def list_retained(max_results: int = 200, timeout: float = 3.0) -> list[dict[str, Any]]:
|
||||
"""Enumerate retained messages (spec section 36).
|
||||
|
||||
MQTT brokers expose no "list retained" API: the defined mechanism is
|
||||
to subscribe to # — the broker immediately delivers every retained
|
||||
message matching the filter (retain flag set on delivery) — collect
|
||||
briefly, then unsubscribe. $SYS is filtered out of the listing.
|
||||
"""
|
||||
settings = get_settings()
|
||||
collected: dict[str, _Message] = {}
|
||||
done = threading.Event()
|
||||
client_id = f"mqtt-explorer-retained-{uuid.uuid4().hex[:8]}"
|
||||
client = mqtt.Client(
|
||||
callback_api_version=CallbackAPIVersion.VERSION2,
|
||||
client_id=client_id,
|
||||
protocol=mqtt.MQTTv5,
|
||||
)
|
||||
client.username_pw_set(
|
||||
settings.mosquitto_dynsec_username, settings.mosquitto_dynsec_password
|
||||
)
|
||||
|
||||
def _on_connect(cl, userdata, flags, reason_code, properties) -> None:
|
||||
if reason_code == 0:
|
||||
cl.subscribe("#", qos=0)
|
||||
else:
|
||||
done.set() # nothing to collect without a subscription
|
||||
|
||||
def _on_message(cl, userdata, msg) -> None:
|
||||
if not getattr(msg, "retain", False):
|
||||
return # only retained deliveries belong in the listing
|
||||
if msg.topic.startswith("$SYS"):
|
||||
return # telemetry noise, not retained app data
|
||||
if len(collected) >= max_results:
|
||||
done.set()
|
||||
return
|
||||
info = _payload_info(msg.payload)
|
||||
collected[msg.topic] = _Message(
|
||||
index=len(collected),
|
||||
topic=msg.topic,
|
||||
qos=msg.qos,
|
||||
retain=True,
|
||||
received_at=time.time(),
|
||||
payload_b64=info["payload_b64"],
|
||||
payload_text=info["payload_text"],
|
||||
)
|
||||
|
||||
client.on_connect = _on_connect
|
||||
client.on_message = _on_message
|
||||
client.connect(settings.mosquitto_host, settings.mosquitto_mgmt_port, keepalive=30)
|
||||
client.loop_start()
|
||||
try:
|
||||
done.wait(timeout)
|
||||
finally:
|
||||
try:
|
||||
client.disconnect()
|
||||
except OSError:
|
||||
pass
|
||||
client.loop_stop()
|
||||
return [m.as_dict() for _, m in sorted(collected.items())]
|
||||
|
||||
|
||||
def publish_message(
|
||||
topic: str, payload: bytes, qos: int, retain: bool, timeout: float = 10.0
|
||||
) -> None:
|
||||
"""Publish a test message as the management identity (spec section 35).
|
||||
|
||||
A short-lived dedicated client keeps the explorer's session clients
|
||||
purely read-only.
|
||||
"""
|
||||
settings = get_settings()
|
||||
client = mqtt.Client(
|
||||
callback_api_version=CallbackAPIVersion.VERSION2,
|
||||
client_id=f"mqtt-explorer-pub-{uuid.uuid4().hex[:8]}",
|
||||
protocol=mqtt.MQTTv5,
|
||||
)
|
||||
client.username_pw_set(
|
||||
settings.mosquitto_dynsec_username, settings.mosquitto_dynsec_password
|
||||
)
|
||||
client.connect(settings.mosquitto_host, settings.mosquitto_mgmt_port, keepalive=30)
|
||||
client.loop_start()
|
||||
try:
|
||||
info = client.publish(topic, payload, qos=qos, retain=retain)
|
||||
info.wait_for_publish(timeout=timeout)
|
||||
if not info.is_published():
|
||||
raise RuntimeError("publish did not complete in time")
|
||||
finally:
|
||||
try:
|
||||
client.disconnect()
|
||||
except OSError:
|
||||
pass
|
||||
client.loop_stop()
|
||||
Reference in New Issue
Block a user