🚧 Move imei_mapping and app_tokens to database

listener/db.py

@@ -2,6 +2,9 @@
 
 The network thread writes rows; writes happen only on that thread so a plain
 (no pool) blocking connection is used, with reconnection on failure.
+
+The IMEI -> device mapping lives in the imei_map table and is polled by a
+separate daemon thread (ImeiMapReader) with its own connection.
 """
 
 from __future__ import annotations
@@ -78,6 +81,87 @@ class Inserter:
                 self._conn.close()
             except psycopg.Error:
                 pass
+
+
+class ImeiMapReader:
+    """Thread that polls the imei_map table for the IMEI -> device mapping.
+
+    Runs its OWN persistent autocommit connection (never shared with
+    Inserter, whose connection is funneled through the MQTT network thread).
+    Reconnects on failure; a failed poll keeps the previously read mapping,
+    mirroring the old broken-file-keeps-last-good-map semantics.
+    """
+
+    POLL_INTERVAL_SECONDS = 5.0
+
+    def __init__(self, dsn: str) -> None:
+        self._dsn = dsn
+        self._lock = threading.Lock()
+        self._conn: Connection | None = None
+        self._imap: dict[str, str] = {}
+        self._stop = threading.Event()
+        self._thread = threading.Thread(
+            target=self._run, name="imei-map-poll", daemon=True
+        )
+
+    def _connection(self) -> Connection:
+        if self._conn is None or self._conn.closed:
+            logger.info("connecting to database (imei map poll)")
+            self._conn = psycopg.connect(self._dsn, autocommit=True)
+        return self._conn
+
+    def read(self) -> dict[str, str]:
+        """Read the mapping from the database (single SELECT).
+
+        Raises psycopg.Error on connection/query failure; the caller keeps
+        the last good mapping."""
+        conn = self._connection()
+        with conn.cursor() as cur:
+            cur.execute("SELECT imei, device FROM imei_map")
+            return dict(cur.fetchall())
+
+    def _run(self) -> None:
+        """Poll loop; runs in its own daemon thread."""
+        while not self._stop.wait(timeout=self.POLL_INTERVAL_SECONDS):
+            try:
+                fresh = self.read()
+            except psycopg.Error:
+                logger.exception("IMEI map poll failed; keeping previous mapping")
+                self._close()
+                continue
+            with self._lock:
+                if fresh != self._imap:
+                    self._imap = fresh
+                    logger.info("IMEI map updated: %d mappings", len(fresh))
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def prime(self, mapping: dict[str, str]) -> None:
+        """Set the initial mapping (call before start())."""
+        with self._lock:
+            self._imap = dict(mapping)
+
+    def snapshot(self) -> dict[str, str]:
+        """Thread-safe copy of the current mapping for the ingest path."""
+        with self._lock:
+            return dict(self._imap)
+
+    def _close(self) -> None:
+        if self._conn is not None:
+            try:
+                self._conn.close()
+            except psycopg.Error:
+                pass
+            self._conn = None
+
+    def close(self) -> None:
+        """Stop the poll thread and close the connection."""
+        self._stop.set()
+        if self._thread.is_alive():
+            self._thread.join(timeout=self.POLL_INTERVAL_SECONDS)
+        with self._lock:
+            self._close()
             self._conn = None
 
     def close(self) -> None:

listener/main.py

@@ -9,7 +9,6 @@ may be retried or dropped depending on QoS.
 
 from __future__ import annotations
 
-import json
 import logging
 import os
 import signal
@@ -17,10 +16,11 @@ import threading
 from pathlib import Path
 
 import paho.mqtt.client as mqtt
+import psycopg
 from paho.mqtt.enums import CallbackAPIVersion
 
-from listener.db import Inserter
-from listener.process import DEFAULT_TIME_KEYS, normalize_imei, process_message, validate_imei_map
+from listener.db import ImeiMapReader, Inserter
+from listener.process import DEFAULT_TIME_KEYS, process_message
 
 logger = logging.getLogger("listener")
 
@@ -28,15 +28,6 @@ _stop_event = threading.Event()
 _shutdown_complete = threading.Event()
 
 
-def load_imei_map(path: str, log: bool = True) -> dict[str, str]:
-    """Load and validate the IMEI -> device name map from a JSON file."""
-    data = json.loads(Path(path).read_text(encoding="utf-8"))
-    result = validate_imei_map(data, path)
-    if log:
-        logger.info("loaded %d IMEI mappings from %s", len(result), path)
-    return result
-
-
 def _env(name: str, default: str | None = None) -> str:
     value = os.environ.get(name)
     if not value:
@@ -61,9 +52,17 @@ def main() -> None:
     topic_filter = _env("MQTT_TOPIC_FILTER", "/+/+")
     qos = int(_env("MQTT_QOS", "1"))
 
-    map_file = _env("IMEI_MAP_FILE")
-    imei_map = load_imei_map(map_file)
-    imei_lock = threading.Lock()  # guards imei_map against the reload thread
+    map_reader = ImeiMapReader(_env("DB_DSN"))
+    # Fail fast when the db (or the imei_map table) is unreachable: without a
+    # mapping the listener would silently store every device name as its IMEI.
+    try:
+        initial_map = map_reader.read()
+    except psycopg.Error:
+        logger.exception("initial IMEI map read failed")
+        raise
+    map_reader.prime(initial_map)
+    map_reader.start()
+    logger.info("loaded %d IMEI mappings from the database", len(initial_map))
     imei_key = _env("INJECT_IMEI_KEY", "imei")
     device_key = _env("INJECT_DEVICE_KEY", "device")
     time_keys = tuple(
@@ -78,25 +77,6 @@ def main() -> None:
     Path("/tmp/listener.pid").write_text(str(os.getpid()), encoding="utf-8")
     logger.info("listener starting, pid %d", os.getpid())
 
-    def reload_imei_map() -> None:
-        """Periodically reload the IMEI map so edits via the API take effect
-        without a container restart. A broken file is logged and skipped; the
-        previously loaded mapping stays in effect."""
-        while not _stop_event.wait(timeout=5.0):
-            try:
-                fresh = load_imei_map(map_file, log=False)
-            except (OSError, ValueError, json.JSONDecodeError):
-                logger.exception("IMEI map reload failed; keeping previous mapping")
-                continue
-            nonlocal imei_map
-            with imei_lock:
-                if fresh != imei_map:
-                    imei_map = fresh
-                    logger.info("IMEI map reloaded: %d mappings", len(fresh))
-
-    reload_thread = threading.Thread(target=reload_imei_map, name="imei-map-reload", daemon=True)
-    reload_thread.start()
-
     def on_connect(client, userdata, flags, reason_code, properties):  # noqa: ANN001
         if reason_code.is_failure:
             logger.error("broker refused connection: %s", reason_code)
@@ -107,12 +87,10 @@ def main() -> None:
     def on_message(client, userdata, message):  # noqa: ANN001
         try:
             topic = message.topic
-            with imei_lock:
-                current_map = imei_map
             results = process_message(
                 topic,
                 bytes(message.payload),
-                current_map,
+                map_reader.snapshot(),
                 imei_key=imei_key,
                 device_key=device_key,
                 time_keys=time_keys,
@@ -157,6 +135,7 @@ def main() -> None:
     client.loop_stop()
     client.disconnect()
     inserter.close()
+    map_reader.close()
     Path("/tmp/listener.pid").unlink(missing_ok=True)
     logger.info("shutdown complete")