jobe/nodeledge-mqtt-listener
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🚧 Add handling of static metadata

Browse Source
This commit is contained in:
Jonas Bengtsson
2026-09-04 22:56:02 +02:00
parent 46639a837c
commit c1b9006f78
9 changed files with 762 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/app.py
+269 -17
View File
@@ -1,6 +1,14 @@
"""FastAPI application exposing read access to the mqtt_messages table and
management of the config tables (app_tokens, imei_map) in the database.
Device settings live in imei_map as one JSONB document per IMEI (the
"device" key holds the device name). They are managed in bulk via
PUT /configs/imei_map or — the way a management frontend should work —
per device via the /devices/{imei} resource endpoints (single-row upsert,
so concurrent edits of different devices never clobber each other).
GET /messages merges each device's CURRENT settings document into the
payload at read time; stored payload keys win on conflicts.
Authentication: app tokens stored as SHA-256 digests in the app_tokens
table, passed either as the `X-App-Token` header or as an `app_token` query
parameter. Digests are compared in constant time; invalid tokens yield 401.
@@ -20,8 +28,9 @@ from typing import Any
import psycopg
from fastapi import Depends, FastAPI, HTTPException, Query, Request, status
from psycopg import sql
from psycopg.types.json import Jsonb
from listener.process import validate_imei_map
from listener.process import normalize_imei, validate_imei_doc, validate_imei_map
logger = logging.getLogger("api")
@@ -156,24 +165,107 @@ class TokenStore:
finally:
conn.close()
def upsert(self, app_name: str, secret: str) -> None:
"""Add or rotate ONE app token in place (single-row upsert).
Unlike replace() this never touches the other apps' rows, so a
token-management UI cannot clobber concurrent edits. Verification
reads the table on every request, so the change takes effect
immediately.
"""
# autocommit: a single-statement write commits immediately (a plain
# connection would roll back on close).
conn = psycopg.connect(self._dsn, autocommit=True)
try:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO app_tokens (app_name, token_hash)
VALUES (%s, %s)
ON CONFLICT (app_name) DO UPDATE
SET token_hash = EXCLUDED.token_hash, updated_at = now()
""",
(app_name, hash_token(secret)),
)
finally:
conn.close()
def delete(self, app_name: str) -> bool:
"""Remove one app token; returns False when the name is unknown."""
conn = psycopg.connect(self._dsn, autocommit=True)
try:
with conn.cursor() as cur:
cur.execute("DELETE FROM app_tokens WHERE app_name = %s", (app_name,))
return cur.rowcount > 0
finally:
conn.close()
class ImeiMapStore:
"""IMEI -> device name mapping backed by the imei_map table."""
"""Device settings backed by the imei_map table: one JSONB document
per IMEI whose "device" key holds the device name (other keys are
arbitrary settings merged into message payloads at read time)."""
def __init__(self, dsn: str) -> None:
self._dsn = dsn
def load(self) -> dict[str, str]:
def load(self) -> dict[str, dict[str, Any]]:
"""All settings documents as {imei: document}."""
conn = psycopg.connect(self._dsn)
try:
with conn.cursor() as cur:
cur.execute("SELECT imei, device FROM imei_map ORDER BY imei")
cur.execute("SELECT imei, meta FROM imei_map ORDER BY imei")
return {row[0]: row[1] for row in cur.fetchall()}
finally:
conn.close()
def replace(self, mapping: dict[str, str]) -> None:
"""Replace the whole mapping (delete + insert) in one transaction."""
def get(self, imei: str) -> dict[str, Any] | None:
"""One settings document, or None when the IMEI is unknown."""
conn = psycopg.connect(self._dsn)
try:
with conn.cursor() as cur:
cur.execute("SELECT meta FROM imei_map WHERE imei = %s", (imei,))
row = cur.fetchone()
return row[0] if row is not None else None
finally:
conn.close()
def upsert(self, imei: str, doc: dict[str, Any]) -> None:
"""Insert or replace ONE device's settings (single-row upsert, so
concurrent edits of different devices never clobber each other)."""
# autocommit: a single-statement write commits immediately (a plain
# connection would roll back on close).
conn = psycopg.connect(self._dsn, autocommit=True)
try:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO imei_map (imei, meta)
VALUES (%s, %s)
ON CONFLICT (imei) DO UPDATE
SET meta = EXCLUDED.meta, updated_at = now()
""",
(imei, Jsonb(doc)),
)
finally:
conn.close()
def delete(self, imei: str) -> bool:
"""Remove one device; returns False when the IMEI is unknown."""
conn = psycopg.connect(self._dsn, autocommit=True)
try:
with conn.cursor() as cur:
cur.execute("DELETE FROM imei_map WHERE imei = %s", (imei,))
return cur.rowcount > 0
finally:
conn.close()
def replace(self, mapping: dict[str, dict[str, Any]]) -> None:
"""Replace ALL devices (delete + insert) in one transaction.
Bulk import only: a management frontend should use the
per-device upsert/delete so concurrent edits of different
devices cannot clobber each other."""
conn = psycopg.connect(self._dsn)
try:
with conn.transaction():
@@ -181,8 +273,8 @@ class ImeiMapStore:
cur.execute("DELETE FROM imei_map")
if mapping:
cur.executemany(
"INSERT INTO imei_map (imei, device) VALUES (%s, %s)",
list(mapping.items()),
"INSERT INTO imei_map (imei, meta) VALUES (%s, %s)",
[(imei, Jsonb(doc)) for imei, doc in mapping.items()],
)
finally:
conn.close()
@@ -207,6 +299,24 @@ def _validate_token_body(data: dict[str, Any], source: str) -> dict[str, str]:
return dict(tokens)
def _validate_single_token_body(data: dict[str, Any], source: str) -> str:
"""Validate a {"token": secret} request body and return the secret.
Raises ValueError with a descriptive message otherwise."""
if (
not isinstance(data, dict)
or not isinstance(data.get("token"), str)
or not data["token"]
):
raise ValueError(f"{source}: expected JSON on the form {{\"token\": \"secret\"}}")
return data["token"]
def _device_repr(imei: str, doc: dict[str, Any]) -> dict[str, Any]:
"""API representation of a device: its settings document plus the
IMEI, so a GET response can be edited and PUT straight back."""
return {"imei": imei, **doc}
def _env(name: str, default: str | None = None) -> str:
value = os.environ.get(name)
if not value:
@@ -319,6 +429,138 @@ def create_app(
raise _db_unavailable(exc) from exc
return {"status": "updated", "config": name}
# --- Device resources: the per-IMEI API a management frontend uses.
# Single-row upsert/delete means editing one device never clobbers a
# concurrent edit of another (unlike the whole-document config PUT).
# The listener picks up changes within ~5 seconds via its poll.
@app.get("/devices", tags=["devices"], dependencies=[Depends(authenticate)])
def list_devices() -> list[dict[str, Any]]:
try:
return [_device_repr(imei, doc) for imei, doc in imei_store.load().items()]
except psycopg.Error as exc:
raise _db_unavailable(exc) from exc
@app.get("/devices/{imei}", tags=["devices"], dependencies=[Depends(authenticate)])
def get_device(imei: str) -> dict[str, Any]:
normalized = normalize_imei(imei)
if normalized is None:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"{imei!r} is not a 15-digit IMEI",
)
try:
doc = imei_store.get(normalized)
except psycopg.Error as exc:
raise _db_unavailable(exc) from exc
if doc is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"unknown device {normalized}",
)
return _device_repr(normalized, doc)
@app.put("/devices/{imei}", tags=["devices"], dependencies=[Depends(authenticate)])
def set_device(imei: str, body: dict[str, Any]) -> dict[str, Any]:
normalized = normalize_imei(imei)
if normalized is None:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"{imei!r} is not a 15-digit IMEI",
)
try:
# A body "imei" key is tolerated when it matches the URL, so a
# GET response can be edited and PUT straight back.
doc = dict(body)
claimed = doc.pop("imei", normalized)
if claimed != normalized:
raise ValueError(
f"request body: imei {claimed!r} does not match the URL imei "
f"{normalized!r}"
)
doc = validate_imei_doc(doc, "request body")
imei_store.upsert(normalized, doc)
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(exc),
) from exc
except psycopg.Error as exc:
raise _db_unavailable(exc) from exc
logger.info("device %s upserted: %d setting(s)", normalized, len(doc))
return _device_repr(normalized, doc)
@app.delete("/devices/{imei}", tags=["devices"], dependencies=[Depends(authenticate)])
def delete_device(imei: str) -> dict[str, Any]:
normalized = normalize_imei(imei)
if normalized is None:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"{imei!r} is not a 15-digit IMEI",
)
try:
deleted = imei_store.delete(normalized)
except psycopg.Error as exc:
raise _db_unavailable(exc) from exc
if not deleted:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"unknown device {normalized}",
)
logger.info("device %s deleted", normalized)
return {"status": "deleted", "imei": normalized}
# --- Token resources: add/rotate/remove ONE app token without
# touching the other apps' rows.
@app.get("/tokens", tags=["tokens"], dependencies=[Depends(authenticate)])
def list_tokens() -> dict[str, Any]:
try:
# Secrets are stored as digests and not recoverable; expose
# only the app names (same shape as GET /configs/app_tokens).
return {"tokens": {name: None for name in token_store.names()}}
except psycopg.Error as exc:
raise _db_unavailable(exc) from exc
@app.put("/tokens/{app_name}", tags=["tokens"], dependencies=[Depends(authenticate)])
def set_token(app_name: str, body: dict[str, Any]) -> dict[str, Any]:
name = app_name.strip()
try:
if not name:
raise ValueError("app name must be a non-empty string")
secret = _validate_single_token_body(body, "request body")
token_store.upsert(name, secret)
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(exc),
) from exc
except psycopg.Error as exc:
raise _db_unavailable(exc) from exc
# The store reads the table on every request: effective immediately.
logger.info("token %r upserted", name)
return {"status": "updated", "app": name}
@app.delete("/tokens/{app_name}", tags=["tokens"], dependencies=[Depends(authenticate)])
def delete_token(app_name: str) -> dict[str, Any]:
name = app_name.strip()
if not name:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="app name must be a non-empty string",
)
try:
deleted = token_store.delete(name)
except psycopg.Error as exc:
raise _db_unavailable(exc) from exc
if not deleted:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"unknown app token {name!r}",
)
logger.info("token %r deleted", name)
return {"status": "deleted", "app": name}
@app.get("/health", tags=["meta"])
def health() -> dict[str, Any]:
return {"status": "ok"}
@@ -394,25 +636,27 @@ def create_app(
detail="'payload_from' must be earlier than or equal to 'payload_to'",
)
limit = min(limit, config.max_limit)
# Conditions reference the aliased mqtt_messages table (m.) because
# the query joins imei_map for the read-time settings merge.
conditions: list[sql.Composable] = []
params: list[Any] = []
if imei:
conditions.append(sql.SQL("imei = %s"))
conditions.append(sql.SQL("m.imei = %s"))
params.append(imei)
if device:
conditions.append(sql.SQL("device = %s"))
conditions.append(sql.SQL("m.device = %s"))
params.append(device)
if from_time is not None:
conditions.append(sql.SQL("received_at >= %s"))
conditions.append(sql.SQL("m.received_at >= %s"))
params.append(_as_utc(from_time))
if to_time is not None:
conditions.append(sql.SQL("received_at <= %s"))
conditions.append(sql.SQL("m.received_at <= %s"))
params.append(_as_utc(to_time))
if payload_from_time is not None:
conditions.append(sql.SQL("payload_time >= %s"))
conditions.append(sql.SQL("m.payload_time >= %s"))
params.append(_as_utc(payload_from_time))
if payload_to_time is not None:
conditions.append(sql.SQL("payload_time <= %s"))
conditions.append(sql.SQL("m.payload_time <= %s"))
params.append(_as_utc(payload_to_time))
where = sql.SQL("")
@@ -422,12 +666,20 @@ def create_app(
# Order by device timestamp when framing on it (NULLS LAST matches the
# idx_mqtt_messages_ptime index); otherwise newest-first by insert id.
if payload_from_time is not None or payload_to_time is not None:
order_by = sql.SQL(" ORDER BY payload_time DESC NULLS LAST LIMIT %s OFFSET %s")
order_by = sql.SQL(" ORDER BY m.payload_time DESC NULLS LAST LIMIT %s OFFSET %s")
else:
order_by = sql.SQL(" ORDER BY id DESC LIMIT %s OFFSET %s")
order_by = sql.SQL(" ORDER BY m.id DESC LIMIT %s OFFSET %s")
# LEFT JOIN imei_map merges the device's CURRENT settings document
# into the payload at read time, so dynamic settings (location,
# thresholds, ...) are up to date for every stored message. jsonb
# concatenation makes the RIGHT operand win on duplicate keys: the
# stored payload (the device's own data plus the ingest-time
# imei/device snapshot) takes precedence over the settings doc.
query = sql.SQL(
"SELECT id, received_at, topic, imei, device, payload, payload_time FROM mqtt_messages"
"SELECT m.id, m.received_at, m.topic, m.imei, m.device, "
"COALESCE(im.meta, '{}'::jsonb) || m.payload, m.payload_time "
"FROM mqtt_messages m LEFT JOIN imei_map im ON im.imei = m.imei"
) + where + order_by
try: