🚧 Move imei_mapping and app_tokens to database

tools/migrate_configs_to_db.py

@@ -0,0 +1,115 @@
+"""One-time migration: app_tokens.json + imei_map.json -> database tables.
+
+Reads the legacy JSON config files, inserts app tokens as SHA-256 digests
+into app_tokens and IMEI -> device mappings into imei_map, then optionally
+verifies the migration with a login round-trip. Run it once against an
+existing deployment after applying init/01_schema.sql (the script is
+idempotent: it truncates both tables first).
+
+Usage:
+    python tools/migrate_configs_to_db.py --dsn postgresql://user:pass@host:5432/db \\
+        --tokens configs/api/app_tokens.json --imei-map configs/listener/imei_map.json \\
+        [--api-url http://localhost:18080] [--api-token <a token from the file>]
+
+Requires psycopg (see tools/requirements.txt). Set --api-url/--api-token to
+verify that the running API authenticates against the migrated tokens.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import psycopg  # noqa: E402
+
+from api.app import hash_token  # noqa: E402
+from listener.process import validate_imei_map  # noqa: E402
+
+
+def load_tokens(path: str) -> dict[str, str]:
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(data, dict) or not isinstance(data.get("tokens"), dict):
+        raise SystemExit(f"{path}: expected JSON on the form {{\"tokens\": {{name: secret}}}}")
+    tokens = data["tokens"]
+    for name, secret in tokens.items():
+        if not isinstance(secret, str) or not secret:
+            raise SystemExit(f"{path}: token {name!r} must be a non-empty string")
+    return dict(tokens)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dsn", required=True, help="PostgreSQL DSN of the stack's db")
+    parser.add_argument("--tokens", required=True, help="legacy app_tokens.json path")
+    parser.add_argument("--imei-map", required=True, help="legacy imei_map.json path")
+    parser.add_argument("--api-url", default=None, help="base URL of the running API for verification")
+    parser.add_argument(
+        "--api-token",
+        default=None,
+        help="one plaintext token from the migrated file, used for verification",
+    )
+    args = parser.parse_args()
+
+    tokens = load_tokens(args.tokens)
+    mapping = validate_imei_map(
+        json.loads(Path(args.imei_map).read_text(encoding="utf-8")), args.imei_map
+    )
+
+    print(f"migrating {len(tokens)} token(s) and {len(mapping)} IMEI mapping(s)")
+    with psycopg.connect(args.dsn) as conn:
+        with conn.transaction():
+            with conn.cursor() as cur:
+                cur.execute("DELETE FROM app_tokens")
+                cur.execute("DELETE FROM imei_map")
+                cur.executemany(
+                    "INSERT INTO app_tokens (app_name, token_hash) VALUES (%s, %s)",
+                    [(name, hash_token(secret)) for name, secret in tokens.items()],
+                )
+                cur.executemany(
+                    "INSERT INTO imei_map (imei, device) VALUES (%s, %s)",
+                    list(mapping.items()),
+                )
+
+    print("migrated. verifying read-back...")
+    with psycopg.connect(args.dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT count(*) FROM app_tokens")
+            row = cur.fetchone()
+            token_count = row[0] if row is not None else 0
+            cur.execute("SELECT count(*) FROM imei_map")
+            row = cur.fetchone()
+            map_count = row[0] if row is not None else 0
+    if token_count != len(tokens) or map_count != len(mapping):
+        print(f"FAIL: expected {len(tokens)}/{len(mapping)}, found {token_count}/{map_count}")
+        return 1
+    print(f"PASS: app_tokens has {token_count} row(s), imei_map has {map_count} row(s)")
+
+    if args.api_url:
+        if not args.api_token:
+            print("--api-token is required with --api-url")
+            return 1
+        req = urllib.request.Request(f"{args.api_url}/messages?limit=1")
+        req.add_header("X-App-Token", args.api_token)
+        try:
+            with urllib.request.urlopen(req, timeout=10) as response:
+                status = response.status
+        except urllib.error.HTTPError as exc:
+            status = exc.code
+        if status == 200:
+            print("PASS: the running API accepts the migrated token")
+        else:
+            print(f"FAIL: API returned {status} for the migrated token (restart the api?)")
+            return 1
+
+    print("done. the JSON files are no longer read by the stack; delete them at will.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/test_listener.py

@@ -1,8 +1,9 @@
-"""Unit tests for listener.process and listener.main config loading."""
+"""Unit tests for listener.process and the API's config/token stores."""
 
 from __future__ import annotations
 
 import json
+import os
 import sys
 from pathlib import Path
 
@@ -10,11 +11,19 @@ import pytest
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
-from listener.main import load_imei_map  # noqa: E402
+# api.app runs create_app() at import time, which requires DB_DSN. It never
+# connects at import (connections are lazy), so a fake DSN is enough; the
+# app_module fixture re-creates the app with injectable in-memory stores.
+os.environ.setdefault(
+    "DB_DSN", "postgresql://user:pass@localhost:5432/db?connect_timeout=2"
+)
+
+from api.app import ImeiMapStore, TokenStore, _validate_token_body, hash_token  # noqa: E402
 from listener.process import (  # noqa: E402
     InjectionResult,
     parse_payload_time,
     process_message,
+    validate_imei_map,
 )
 from datetime import datetime, timezone  # noqa: E402
 
@@ -256,112 +265,174 @@ class TestPayloadTimeExtraction:
         assert result[0].payload_time is None
 
 
-class TestLoadImeiMap:
-    def test_loads_valid_map(self, tmp_path: Path) -> None:
-        path = tmp_path / "map.json"
-        path.write_text(
-            json.dumps({"350123456789012": "Device X"}), encoding="utf-8"
-        )
-        loaded = load_imei_map(str(path))
-        assert loaded == {"350123456789012": "Device X"}
+class TestValidateImeiMap:
+    """validate_imei_map is the shared schema check for the imei_map table
+    writes (API request bodies and the migration tool)."""
 
-    def test_rejects_non_object(self, tmp_path: Path) -> None:
-        path = tmp_path / "map.json"
-        path.write_text("[]", encoding="utf-8")
+    def test_accepts_valid_map(self) -> None:
+        assert validate_imei_map(
+            {"350123456789012": "Device X"}, "test"
+        ) == {"350123456789012": "Device X"}
+
+    def test_rejects_non_object(self) -> None:
         with pytest.raises(ValueError, match="expected a JSON object"):
-            load_imei_map(str(path))
+            validate_imei_map([], "test")
 
-    def test_rejects_non_string_values(self, tmp_path: Path) -> None:
-        path = tmp_path / "map.json"
-        path.write_text(json.dumps({"350123456789012": 42}), encoding="utf-8")
+    def test_rejects_non_string_values(self) -> None:
         with pytest.raises(ValueError, match="non-empty strings"):
-            load_imei_map(str(path))
+            validate_imei_map({"350123456789012": 42}, "test")
 
-    def test_rejects_non_imei_key(self, tmp_path: Path) -> None:
-        path = tmp_path / "map.json"
+    def test_rejects_non_imei_key(self) -> None:
         # Old dev EUI-style key: no longer accepted
-        path.write_text(json.dumps({"0011223344556677": "Device X"}), encoding="utf-8")
         with pytest.raises(ValueError, match="15-digit IMEI"):
-            load_imei_map(str(path))
+            validate_imei_map({"0011223344556677": "Device X"}, "test")
+
+
+class TestTokenHelpers:
+    def test_hash_token_is_sha256_hex(self) -> None:
+        digest = hash_token("secret-abc")
+        assert len(digest) == 64
+        int(digest, 16)  # hex digest
+
+    def test_hash_token_is_deterministic(self) -> None:
+        assert hash_token("secret-abc") == hash_token("secret-abc")
+        assert hash_token("secret-abc") != hash_token("secret-abd")
+
+    def test_validate_token_body_extracts_tokens(self) -> None:
+        tokens = _validate_token_body({"tokens": {"app": "secret"}}, "test")
+        assert tokens == {"app": "secret"}
+
+    def test_validate_token_body_rejects_wrong_shape(self) -> None:
+        with pytest.raises(ValueError, match="expected JSON on the form"):
+            _validate_token_body({"foo": "bar"}, "test")
+
+    def test_validate_token_body_rejects_empty_secret(self) -> None:
+        with pytest.raises(ValueError, match="non-empty string"):
+            _validate_token_body({"tokens": {"app": ""}}, "test")
+
+
+class FakeTokenStore(TokenStore):
+    """In-memory token store: no database, immediate verification."""
+
+    def __init__(self, tokens: dict[str, str]) -> None:
+        super().__init__("postgresql://unused")
+        self._tokens = dict(tokens)
+        self.replaced_with: dict[str, str] | None = None
+
+    def verify(self, token: str) -> bool:
+        return token in self._tokens.values()
+
+    def names(self) -> list[str]:
+        return sorted(self._tokens)
+
+    def replace(self, tokens: dict[str, str]) -> None:
+        self.replaced_with = dict(tokens)
+        self._tokens = dict(tokens)
+
+
+class FakeImeiMapStore(ImeiMapStore):
+    """In-memory imei map store: no database."""
+
+    def __init__(self, mapping: dict[str, str]) -> None:
+        super().__init__("postgresql://unused")
+        self._mapping = dict(mapping)
+        self.replaced_with: dict[str, str] | None = None
+
+    def load(self) -> dict[str, str]:
+        return dict(self._mapping)
+
+    def replace(self, mapping: dict[str, str]) -> None:
+        self.replaced_with = dict(mapping)
+        self._mapping = dict(mapping)
 
 
 @pytest.fixture()
-def app_module(tmp_path):
-    """Import api.app with a prepared environment (create_app runs at import
-    time). Yields the reloaded module and restores the environment after."""
+def app_module(monkeypatch):
+    """Import api.app with a fake environment (create_app runs at import
+    time; DB connections are lazy so nothing connects yet). monkeypatch
+    restores the environment automatically."""
     import importlib
-    import os
 
-    # connect_timeout keeps the DB-connecting tests fast even when
+    # Legacy file-based env vars must NOT be set for the reloaded module.
+    for legacy in ("APP_TOKENS_FILE", "IMEI_MAP_FILE"):
+        monkeypatch.delenv(legacy, raising=False)
+    # connect_timeout keeps DB-connecting tests fast even when
     # localhost:5432 is silently dropped by a firewall (psycopg otherwise
     # waits out the OS-level TCP timeout).
-    tokens = tmp_path / "app_tokens.json"
-    tokens.write_text(
-        json.dumps({"tokens": {"test-app": "secret-abc"}}), encoding="utf-8"
+    monkeypatch.setenv(
+        "DB_DSN", "postgresql://user:pass@localhost:5432/db?connect_timeout=2"
     )
-    imei_map = tmp_path / "imei_map.json"
-    imei_map.write_text(
-        json.dumps({"350123456789012": "Greenhouse sensor A"}), encoding="utf-8"
-    )
-    env = {
-        "APP_TOKENS_FILE": str(tokens),
-        "IMEI_MAP_FILE": str(imei_map),
-        "DB_DSN": "postgresql://user:pass@localhost:5432/db?connect_timeout=2",
-    }
-    old = {key: os.environ.get(key) for key in env}
-    os.environ.update(env)
-    try:
-        import api.app as app_module
+    import api.app as app_module
 
-        importlib.reload(app_module)
-        yield app_module
-    finally:
-        for key, value in old.items():
-            if value is None:
-                os.environ.pop(key, None)
-            else:
-                os.environ[key] = value
+    importlib.reload(app_module)
+    yield app_module
+
+
+@pytest.fixture()
+def api(app_module):
+    """ASGI app wired to in-memory fakes; yields (app, token_store, imei_store)."""
+    tokens = FakeTokenStore({"test-app": "secret-abc"})
+    imeis = FakeImeiMapStore({"350123456789012": "Greenhouse sensor A"})
+    app = app_module.create_app(token_store=tokens, imei_store=imeis)
+    return app, tokens, imeis
 
 
 class TestApiAuth:
     """Light tests of the API auth dependency using the ASGI app directly."""
 
-    def test_health_is_public(self, app_module) -> None:
+    def test_health_is_public(self, api) -> None:
         from fastapi.testclient import TestClient
 
-        client = TestClient(app_module.app)
-        assert client.get("/health").status_code == 405 or client.get("/health").json() == {"status": "ok"}
+        client = TestClient(api[0])
+        assert client.get("/health").json() == {"status": "ok"}
 
-    def test_missing_token_rejected(self, app_module) -> None:
+    def test_missing_token_rejected(self, api) -> None:
         from fastapi.testclient import TestClient
 
-        client = TestClient(app_module.app)
+        client = TestClient(api[0])
         response = client.get("/messages?limit=1")
         assert response.status_code == 401
 
-    def test_wrong_token_rejected(self, app_module) -> None:
+    def test_wrong_token_rejected(self, api) -> None:
         from fastapi.testclient import TestClient
 
-        client = TestClient(app_module.app)
+        client = TestClient(api[0])
         response = client.get("/messages?limit=1", headers={"X-App-Token": "nope"})
         assert response.status_code == 401
 
-    def test_valid_token_accepted(self, app_module) -> None:
+    def test_valid_token_accepted(self, api) -> None:
         from fastapi.testclient import TestClient
 
         # The DB is not reachable in unit tests, so a valid token proceeds past
         # auth and then fails on the database connection with a 500. That
         # proves authentication passed (401 would mean it did not).
-        client = TestClient(app_module.app, raise_server_exceptions=False)
+        client = TestClient(api[0], raise_server_exceptions=False)
         response = client.get("/messages?limit=1", headers={"X-App-Token": "secret-abc"})
         assert response.status_code != 401
 
-    def test_reversed_time_frame_rejected(self, app_module) -> None:
+    def test_db_outage_yields_503_not_500(self, app_module) -> None:
+        from fastapi.testclient import TestClient
+
+        import psycopg
+
+        class BrokenStore(FakeTokenStore):
+            def verify(self, token: str) -> bool:
+                raise psycopg.OperationalError("db is down")
+
+        app = app_module.create_app(
+            token_store=BrokenStore({"test-app": "secret-abc"}),
+            imei_store=FakeImeiMapStore({}),
+        )
+        client = TestClient(app, raise_server_exceptions=False)
+        response = client.get("/messages?limit=1", headers={"X-App-Token": "secret-abc"})
+        assert response.status_code == 503
+
+    def test_reversed_time_frame_rejected(self, api) -> None:
         from fastapi.testclient import TestClient
 
         # Range validation runs before any database access, so this works
         # without a reachable DB.
-        client = TestClient(app_module.app)
+        client = TestClient(api[0])
         response = client.get(
             "/messages",
             params={"from": "2026-08-31T12:00:00Z", "to": "2026-08-31T11:00:00Z"},
@@ -369,12 +440,12 @@ class TestApiAuth:
         )
         assert response.status_code == 422
 
-    def test_naive_and_offset_timestamps_accepted(self, app_module) -> None:
+    def test_naive_and_offset_timestamps_accepted(self, api) -> None:
         from fastapi.testclient import TestClient
 
         # Both forms must parse; the request fails later on DB access (500),
         # never with a 422 validation error.
-        client = TestClient(app_module.app, raise_server_exceptions=False)
+        client = TestClient(api[0], raise_server_exceptions=False)
         for value in ("2026-08-31T12:00:00", "2026-08-31T12:00:00+02:00"):
             response = client.get(
                 "/messages",
@@ -385,85 +456,96 @@ class TestApiAuth:
 
 
 class TestConfigEndpoints:
-    """GET/PUT/POST /configs/{name} — reads, validated JSON writes."""
-
-    @pytest.fixture()
-    def client(self, app_module):
-        from fastapi.testclient import TestClient
-
-        return TestClient(app_module.app)
+    """GET/PUT/POST /configs/{name} — reads, validated writes into stores."""
 
     HDRS = {"X-App-Token": "secret-abc"}
 
-    def test_get_imei_map(self, client, app_module) -> None:
+    @pytest.fixture()
+    def client_and_stores(self, api):
+        from fastapi.testclient import TestClient
+
+        app, tokens, imeis = api
+        return TestClient(app), tokens, imeis
+
+    def test_get_imei_map(self, client_and_stores) -> None:
+        client, _, imeis = client_and_stores
         r = client.get("/configs/imei_map", headers=self.HDRS)
         assert r.status_code == 200
         assert r.json() == {"350123456789012": "Greenhouse sensor A"}
+        assert imeis.load() == {"350123456789012": "Greenhouse sensor A"}
 
-    def test_get_app_tokens(self, client) -> None:
+    def test_get_app_tokens_masks_secrets(self, client_and_stores) -> None:
+        # secrets are stored as digests and not recoverable; only names show
+        client, _, _ = client_and_stores
         r = client.get("/configs/app_tokens", headers=self.HDRS)
         assert r.status_code == 200
-        assert r.json() == {"tokens": {"test-app": "secret-abc"}}
+        assert r.json() == {"tokens": {"test-app": None}}
 
-    def test_get_unknown_config_404(self, client) -> None:
+    def test_get_unknown_config_404(self, client_and_stores) -> None:
+        client, _, _ = client_and_stores
         r = client.get("/configs/does-not-exist", headers=self.HDRS)
         assert r.status_code == 404
 
-    def test_config_endpoints_require_token(self, client) -> None:
+    def test_config_endpoints_require_token(self, client_and_stores) -> None:
+        client, _, _ = client_and_stores
         assert client.get("/configs/imei_map").status_code == 401
         assert client.put("/configs/imei_map", json={}).status_code == 401
 
-    def test_put_valid_imei_map(self, client, app_module) -> None:
-        import os
-
+    def test_put_valid_imei_map(self, client_and_stores) -> None:
+        client, _, imeis = client_and_stores
         body = {"350123456789012": "Greenhouse sensor A", "490154203237518": "Water tank sensor B"}
         r = client.put("/configs/imei_map", json=body, headers=self.HDRS)
         assert r.status_code == 200
         assert r.json() == {"status": "updated", "config": "imei_map"}
-        # file on disk holds the new content
-        with open(os.environ["IMEI_MAP_FILE"], encoding="utf-8") as f:
-            assert json.load(f) == body
+        # the store (and thereby the table) holds the new content
+        assert imeis.replaced_with == body
 
-    def test_post_is_alias_for_put(self, client) -> None:
+    def test_post_is_alias_for_put(self, client_and_stores) -> None:
+        client, _, _ = client_and_stores
         body = {"350123456789012": "Greenhouse sensor A"}
         r = client.post("/configs/imei_map", json=body, headers=self.HDRS)
         assert r.status_code == 200
 
-    def test_put_rejects_non_imei_keys(self, client) -> None:
+    def test_put_rejects_non_imei_keys(self, client_and_stores) -> None:
+        client, _, _ = client_and_stores
         r = client.put("/configs/imei_map", json={"0011223344556677": "X"}, headers=self.HDRS)
         assert r.status_code == 422
         assert "15-digit IMEI" in r.json()["detail"]
 
-    def test_put_rejects_non_string_values(self, client) -> None:
+    def test_put_rejects_non_string_values(self, client_and_stores) -> None:
+        client, _, _ = client_and_stores
         r = client.put("/configs/imei_map", json={"350123456789012": 42}, headers=self.HDRS)
         assert r.status_code == 422
 
-    def test_put_rejects_bad_token_file_shape(self, client) -> None:
+    def test_put_rejects_bad_token_body_shape(self, client_and_stores) -> None:
         # JSON object, but not the {"tokens": {...}} shape
+        client, _, _ = client_and_stores
         r = client.put("/configs/app_tokens", json={"foo": "bar"}, headers=self.HDRS)
         assert r.status_code == 422
 
-    def test_put_rejects_empty_token_secret(self, client) -> None:
+    def test_put_rejects_empty_token_secret(self, client_and_stores) -> None:
+        client, _, _ = client_and_stores
         r = client.put(
             "/configs/app_tokens", json={"tokens": {"app": ""}}, headers=self.HDRS
         )
         assert r.status_code == 422
 
-    def test_non_object_body_rejected_by_fastapi(self, client) -> None:
+    def test_non_object_body_rejected_by_fastapi(self, client_and_stores) -> None:
         # JSON but not an object: FastAPI's dict[str, Any] annotation rejects
         # arrays and scalars before the handler runs
+        client, _, _ = client_and_stores
         r = client.put("/configs/imei_map", json=[1, 2, 3], headers=self.HDRS)
         assert r.status_code == 422
         r = client.put("/configs/imei_map", json="just a string", headers=self.HDRS)
         assert r.status_code == 422
 
-    def test_app_tokens_update_reloads_auth(self, client, app_module) -> None:
+    def test_app_tokens_update_reloads_auth(self, client_and_stores) -> None:
         # rotate the token, then authenticate with the NEW one
+        client, tokens, _ = client_and_stores
         new_tokens = {"tokens": {"test-app": "secret-abc", "rotated": "fresh-secret"}}
         r = client.put("/configs/app_tokens", json=new_tokens, headers=self.HDRS)
         assert r.status_code == 200
-        r = client.get("/configs/app_tokens", headers={"X-App-Token": "fresh-secret"})
-        assert r.status_code == 200
-        # the old token still works too (it was kept in the file)
-        r = client.get("/configs/app_tokens", headers=self.HDRS)
-        assert r.status_code == 200
+        # the store verifies against the replaced set immediately
+        assert tokens.verify("fresh-secret")
+        assert tokens.verify("secret-abc")
+        assert not tokens.verify("nope")