🚧 Move imei_mapping and app_tokens to database

api/app.py

@@ -1,20 +1,20 @@
 """FastAPI application exposing read access to the mqtt_messages table and
-management of the stack's JSON config files.
+management of the config tables (app_tokens, imei_map) in the database.
 
-Authentication: static app tokens from a JSON config file, passed either as
-the `X-App-Token` header or as an `app_token` query parameter. Tokens are
-compared in constant time; invalid tokens yield 401.
+Authentication: app tokens stored as SHA-256 digests in the app_tokens
+table, passed either as the `X-App-Token` header or as an `app_token` query
+parameter. Digests are compared in constant time; invalid tokens yield 401.
+When the database is unavailable, authenticated endpoints return 503.
 """
 
 from __future__ import annotations
 
+import hashlib
 import hmac
-import json
 import logging
 import os
-import tempfile
+import threading
 from datetime import datetime, timezone
-from pathlib import Path
 from typing import Any
 
 import psycopg
@@ -26,39 +26,172 @@ from listener.process import validate_imei_map
 logger = logging.getLogger("api")
 
 
-def _atomic_write(path: Path, content: str) -> None:
-    """Write content to path atomically: tempfile in the same directory, then
-    os.replace, so concurrent readers never observe a torn or empty file."""
-    fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp")
-    try:
-        with os.fdopen(fd, "w", encoding="utf-8") as f:
-            f.write(content)
-            f.flush()
-            os.fsync(f.fileno())
-        os.replace(tmp_name, path)
-    except Exception:
-        try:
-            os.unlink(tmp_name)
-        except OSError:
-            pass
-        raise
+def hash_token(token: str) -> str:
+    """SHA-256 hex digest of an app token. Only digests are stored in the
+    database; plaintext tokens are never persisted."""
+    return hashlib.sha256(token.encode("utf-8")).hexdigest()
+
+
+def _db_unavailable(exc: Exception) -> HTTPException:
+    """503 for database outages: without the db the API has no purpose."""
+    return HTTPException(
+        status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+        detail=f"database unavailable: {exc}",
+    )
 
 
 class AppConfig:
     """Runtime configuration loaded once at startup."""
 
-    def __init__(self, dsn: str, tokens: dict[str, str], max_limit: int) -> None:
+    def __init__(self, dsn: str, max_limit: int) -> None:
         self.dsn = dsn
-        self.tokens = tokens
         self.max_limit = max_limit
 
 
-def load_tokens(path: str) -> dict[str, str]:
-    """Load {"tokens": {"app-name": "secret"}} from a JSON file."""
-    return _extract_tokens(json.loads(Path(path).read_text(encoding="utf-8")), path)
+class TokenStore:
+    """API token store backed by the app_tokens table (SHA-256 digests).
+
+    A bootstrap token (BOOTSTRAP_APP_TOKEN) is seeded lazily on the first
+    authentication attempt when the table is empty, so a fresh deployment
+    can authenticate without direct database access. Verification reads the
+    table on every request, so token updates take effect immediately.
+    """
+
+    def __init__(
+        self,
+        dsn: str,
+        bootstrap_token: str | None = None,
+        bootstrap_app: str = "bootstrap",
+    ) -> None:
+        self._dsn = dsn
+        self._bootstrap_token = bootstrap_token
+        self._bootstrap_app = bootstrap_app
+        self._bootstrap_lock = threading.Lock()
+        self._bootstrapped = False
+
+    def _ensure_bootstrapped(self) -> None:
+        """Seed the bootstrap token once, if configured and the table is
+        empty. Runs lazily (on the first authenticate call) so a database
+        outage at startup does not prevent the API from starting; a failed
+        attempt is retried on the next request."""
+        if self._bootstrapped:
+            return
+        with self._bootstrap_lock:
+            if self._bootstrapped:
+                return
+            conn = psycopg.connect(self._dsn, autocommit=True)
+            try:
+                with conn.cursor() as cur:
+                    if self._bootstrap_token:
+                        # Seed only when the table is empty; ON CONFLICT
+                        # covers a race with another worker doing the same.
+                        cur.execute(
+                            """
+                            INSERT INTO app_tokens (app_name, token_hash)
+                            SELECT %s, %s
+                            WHERE NOT EXISTS (SELECT 1 FROM app_tokens)
+                            ON CONFLICT DO NOTHING
+                            """,
+                            (self._bootstrap_app, hash_token(self._bootstrap_token)),
+                        )
+                        if cur.rowcount:
+                            logger.info(
+                                "seeded bootstrap token %r from BOOTSTRAP_APP_TOKEN",
+                                self._bootstrap_app,
+                            )
+                    else:
+                        cur.execute("SELECT count(*) FROM app_tokens")
+                        if cur.fetchone()[0] == 0:
+                            logger.warning(
+                                "app_tokens is empty and BOOTSTRAP_APP_TOKEN is not "
+                                "set; the API will reject all requests until a token "
+                                "is added"
+                            )
+            finally:
+                conn.close()
+            self._bootstrapped = True
+
+    def verify(self, token: str) -> bool:
+        """Constant-time check of a presented token against all digests."""
+        self._ensure_bootstrapped()
+        digest = hash_token(token)
+        for stored in self._hashes().values():
+            if hmac.compare_digest(stored, digest):
+                return True
+        return False
+
+    def _hashes(self) -> dict[str, str]:
+        """All rows as {app_name: token_hash}."""
+        conn = psycopg.connect(self._dsn)
+        try:
+            with conn.cursor() as cur:
+                cur.execute("SELECT app_name, token_hash FROM app_tokens")
+                return dict(cur.fetchall())
+        finally:
+            conn.close()
+
+    def names(self) -> list[str]:
+        """App names with a token (secrets are not recoverable)."""
+        conn = psycopg.connect(self._dsn)
+        try:
+            with conn.cursor() as cur:
+                cur.execute("SELECT app_name FROM app_tokens ORDER BY app_name")
+                return [row[0] for row in cur.fetchall()]
+        finally:
+            conn.close()
+
+    def replace(self, tokens: dict[str, str]) -> None:
+        """Replace ALL tokens (delete + insert) in one transaction."""
+        conn = psycopg.connect(self._dsn)
+        try:
+            with conn.transaction():
+                with conn.cursor() as cur:
+                    cur.execute("DELETE FROM app_tokens")
+                    if tokens:
+                        cur.executemany(
+                            "INSERT INTO app_tokens (app_name, token_hash) "
+                            "VALUES (%s, %s)",
+                            [(name, hash_token(secret)) for name, secret in tokens.items()],
+                        )
+        finally:
+            conn.close()
 
 
-def _extract_tokens(data: object, source: str) -> dict[str, str]:
+class ImeiMapStore:
+    """IMEI -> device name mapping backed by the imei_map table."""
+
+    def __init__(self, dsn: str) -> None:
+        self._dsn = dsn
+
+    def load(self) -> dict[str, str]:
+        conn = psycopg.connect(self._dsn)
+        try:
+            with conn.cursor() as cur:
+                cur.execute("SELECT imei, device FROM imei_map ORDER BY imei")
+                return {row[0]: row[1] for row in cur.fetchall()}
+        finally:
+            conn.close()
+
+    def replace(self, mapping: dict[str, str]) -> None:
+        """Replace the whole mapping (delete + insert) in one transaction."""
+        conn = psycopg.connect(self._dsn)
+        try:
+            with conn.transaction():
+                with conn.cursor() as cur:
+                    cur.execute("DELETE FROM imei_map")
+                    if mapping:
+                        cur.executemany(
+                            "INSERT INTO imei_map (imei, device) VALUES (%s, %s)",
+                            list(mapping.items()),
+                        )
+        finally:
+            conn.close()
+
+
+def _validate_token_body(data: dict[str, Any], source: str) -> dict[str, str]:
+    """Validate a {"tokens": {app: secret}} request body and return the
+    inner tokens mapping. Raises ValueError with a descriptive message
+    otherwise."""
     if not isinstance(data, dict) or not isinstance(data.get("tokens"), dict):
         raise ValueError(f"{source}: expected JSON on the form {{\"tokens\": {{name: secret}}}}")
     tokens = data["tokens"]
@@ -66,7 +199,11 @@ def _extract_tokens(data: object, source: str) -> dict[str, str]:
         if not isinstance(secret, str) or not secret:
             raise ValueError(f"{source}: token {name!r} must be a non-empty string")
     if not tokens:
-        logger.warning("%s contains no tokens; the API will reject all requests", source)
+        logger.warning(
+            "%s replaces app_tokens with an empty set; the API will reject "
+            "all requests until a token is added again",
+            source,
+        )
     return dict(tokens)
 
 
@@ -84,30 +221,33 @@ def _as_utc(value: datetime) -> datetime:
     return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
 
 
-def create_app() -> FastAPI:
+def create_app(
+    token_store: TokenStore | None = None,
+    imei_store: ImeiMapStore | None = None,
+) -> FastAPI:
+    """Build the API application.
+
+    The stores are injectable so tests can run without a database; in
+    production the defaults are built from the environment (DB_DSN,
+    BOOTSTRAP_APP_TOKEN, BOOTSTRAP_APP_NAME).
+    """
     app = FastAPI(
         title="nodeledge MQTT data API",
         version="1.0.0",
         description="Read access to MQTT-collected JSON payloads.",
     )
 
-    tokens_path = _env("APP_TOKENS_FILE")
     dsn = _env("DB_DSN")
     max_limit = int(_env("API_MAX_LIMIT", "10000"))
-    config = AppConfig(dsn=dsn, tokens=load_tokens(tokens_path), max_limit=max_limit)
-
-    # Whitelisted config files the /configs endpoints may read and write. Only
-    # these names resolve; anything else 404s so the API cannot touch files
-    # outside the mounted config directories. Validators enforce each file's
-    # schema on write (a JSON object plus per-config rules).
-    config_files: dict[str, Path] = {
-        "app_tokens": Path(tokens_path),
-        "imei_map": Path(_env("IMEI_MAP_FILE", "/app/config/imei_map.json")),
-    }
-    config_validators: dict[str, Any] = {
-        "app_tokens": lambda body: _extract_tokens(body, "request body"),
-        "imei_map": lambda body: validate_imei_map(body, "request body"),
-    }
+    config = AppConfig(dsn=dsn, max_limit=max_limit)
+    if token_store is None:
+        token_store = TokenStore(
+            dsn,
+            bootstrap_token=os.environ.get("BOOTSTRAP_APP_TOKEN") or None,
+            bootstrap_app=os.environ.get("BOOTSTRAP_APP_NAME") or "bootstrap",
+        )
+    if imei_store is None:
+        imei_store = ImeiMapStore(dsn)
 
     def authenticate(request: Request) -> None:
         token = request.headers.get("X-App-Token")
@@ -118,146 +258,83 @@ def create_app() -> FastAPI:
                 status_code=status.HTTP_401_UNAUTHORIZED,
                 detail="missing app token: pass the X-App-Token header",
             )
-        for candidate in config.tokens.values():
-            if hmac.compare_digest(candidate, token):
-                return
-        raise HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED,
-            detail="invalid app token",
-        )
+        try:
+            if not token_store.verify(token):
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="invalid app token",
+                )
+        except psycopg.Error as exc:
+            raise _db_unavailable(exc) from exc
 
     app.state.config = config
     app.state.authenticate = authenticate
 
+    # Whitelisted config names the /configs endpoints may touch; anything
+    # else 404s. Validators enforce each table's schema on write.
+    config_names = ("app_tokens", "imei_map")
+
+    def _unknown_config(name: str) -> HTTPException:
+        return HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"unknown config {name!r}; available: {', '.join(config_names)}",
+        )
+
     @app.get("/configs/{name}", tags=["config"], dependencies=[Depends(authenticate)])
     def get_config(name: str) -> dict[str, Any]:
-        path = config_files.get(name)
-        if path is None:
-            raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail=f"unknown config {name!r}; available: {', '.join(sorted(config_files))}",
-            )
         try:
-            content = path.read_text(encoding="utf-8")
-        except FileNotFoundError:
-            raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail=f"config file {name!r} not found on disk",
-            )
-        try:
-            return json.loads(content)
-        except json.JSONDecodeError as exc:
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=f"config file {name!r} contains invalid JSON: {exc}",
-            )
-
-    def _set_config(name: str, body: dict[str, Any]) -> dict[str, Any]:
-        path = config_files.get(name)
-        if path is None:
-            raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail=f"unknown config {name!r}; available: {', '.join(sorted(config_files))}",
-            )
-        validator = config_validators.get(name)
-        if validator is not None:
-            try:
-                validator(body)
-            except ValueError as exc:
-                raise HTTPException(
-                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                    detail=str(exc),
-                )
-        try:
-            _atomic_write(path, json.dumps(body, indent=2, ensure_ascii=False) + "\n")
-        except OSError as exc:
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=f"failed to write config {name!r}: {exc}",
-            )
-        # app_tokens backs this API's own authentication: reload immediately
-        # so callers are not stuck with the old tokens after an update.
-        if name == "app_tokens":
-            config.tokens = load_tokens(str(path))
-            logger.info("app tokens reloaded after config update")
-        return {"status": "updated", "config": name}
+            if name == "app_tokens":
+                # Secrets are stored as digests and not recoverable; expose
+                # only the app names so clients can see what exists.
+                return {"tokens": {app: None for app in token_store.names()}}
+            if name == "imei_map":
+                return imei_store.load()
+            raise _unknown_config(name)
+        except psycopg.Error as exc:
+            raise _db_unavailable(exc) from exc
 
     @app.put("/configs/{name}", tags=["config"], dependencies=[Depends(authenticate)])
     @app.post("/configs/{name}", tags=["config"], dependencies=[Depends(authenticate)])
     def set_config(name: str, body: dict[str, Any]) -> dict[str, Any]:
-        return _set_config(name, body)
-
-    @app.get("/configs/{name}", tags=["config"], dependencies=[Depends(authenticate)])
-    def get_config(name: str) -> dict[str, Any]:
-        path = config_files.get(name)
-        if path is None:
-            raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail=f"unknown config {name!r}; available: {', '.join(sorted(config_files))}",
-            )
         try:
-            content = path.read_text(encoding="utf-8")
-        except FileNotFoundError:
+            if name == "app_tokens":
+                tokens = _validate_token_body(body, "request body")
+                token_store.replace(tokens)
+                # The store reads the table on every request, so the new
+                # tokens take effect immediately - no reload needed.
+                logger.info("app_tokens replaced: %d token(s)", len(tokens))
+            elif name == "imei_map":
+                mapping = validate_imei_map(body, "request body")
+                imei_store.replace(mapping)
+                # The listener polls the imei_map table every 5 seconds.
+                logger.info("imei_map replaced: %d mapping(s)", len(mapping))
+            else:
+                raise _unknown_config(name)
+        except ValueError as exc:
             raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail=f"config file {name!r} not found on disk",
-            )
-        try:
-            return json.loads(content)
-        except json.JSONDecodeError as exc:
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=f"config file {name!r} contains invalid JSON: {exc}",
-            )
-
-    def _set_config(name: str, body: dict[str, Any]) -> dict[str, Any]:
-        path = config_files.get(name)
-        if path is None:
-            raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail=f"unknown config {name!r}; available: {', '.join(sorted(config_files))}",
-            )
-        validator = config_validators.get(name)
-        if validator is not None:
-            try:
-                validator(body)
-            except ValueError as exc:
-                raise HTTPException(
-                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                    detail=str(exc),
-                )
-        try:
-            _atomic_write(path, json.dumps(body, indent=2, ensure_ascii=False) + "\n")
-        except OSError as exc:
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=f"failed to write config {name!r}: {exc}",
-            )
-        # app_tokens backs this API's own authentication: reload immediately
-        # so callers are not stuck with the old tokens after an update.
-        if name == "app_tokens":
-            config.tokens = load_tokens(str(path))
-            logger.info("app tokens reloaded after config update")
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=str(exc),
+            ) from exc
+        except psycopg.Error as exc:
+            raise _db_unavailable(exc) from exc
         return {"status": "updated", "config": name}
 
-    @app.put("/configs/{name}", tags=["config"], dependencies=[Depends(authenticate)])
-    @app.post("/configs/{name}", tags=["config"], dependencies=[Depends(authenticate)])
-    def set_config(name: str, body: dict[str, Any]) -> dict[str, Any]:
-        return _set_config(name, body)
-
     @app.get("/health", tags=["meta"])
     def health() -> dict[str, Any]:
         return {"status": "ok"}
 
     @app.get("/messages/count", tags=["meta"], dependencies=[Depends(authenticate)])
     def message_count() -> dict[str, int]:
-        conn = psycopg.connect(config.dsn)
         try:
-            with conn.cursor() as cur:
-                cur.execute("SELECT count(*) FROM mqtt_messages")
-                row = cur.fetchone()
-        finally:
-            conn.close()
+            conn = psycopg.connect(config.dsn)
+            try:
+                with conn.cursor() as cur:
+                    cur.execute("SELECT count(*) FROM mqtt_messages")
+                    row = cur.fetchone()
+            finally:
+                conn.close()
+        except psycopg.Error as exc:
+            raise _db_unavailable(exc) from exc
         count = row[0] if row is not None else 0
         return {"count": count}
 
@@ -353,13 +430,16 @@ def create_app() -> FastAPI:
             "SELECT id, received_at, topic, imei, device, payload, payload_time FROM mqtt_messages"
         ) + where + order_by
 
-        conn = psycopg.connect(config.dsn)
         try:
-            with conn.cursor() as cur:
-                cur.execute(query, (*params, limit, offset))
-                rows = cur.fetchall()
-        finally:
-            conn.close()
+            conn = psycopg.connect(config.dsn)
+            try:
+                with conn.cursor() as cur:
+                    cur.execute(query, (*params, limit, offset))
+                    rows = cur.fetchall()
+            finally:
+                conn.close()
+        except psycopg.Error as exc:
+            raise _db_unavailable(exc) from exc
 
         return {
             "limit": limit,