🚧 Add handling of static metadata

tools/migrate_configs_to_db.py

@@ -1,8 +1,10 @@
 """One-time migration: app_tokens.json + imei_map.json -> database tables.
 
 Reads the legacy JSON config files, inserts app tokens as SHA-256 digests
-into app_tokens and IMEI -> device mappings into imei_map, then optionally
-verifies the migration with a login round-trip. Run it once against an
+into app_tokens and device settings documents into imei_map (flat
+"imei": "device name" values are promoted to {"device": ...}), then
+optionally verifies the migration with a login round-trip. Run it once
+against an
 existing deployment after applying init/01_schema.sql (the script is
 idempotent: it truncates both tables first).
 
@@ -27,6 +29,7 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
 import psycopg  # noqa: E402
+from psycopg.types.json import Jsonb  # noqa: E402
 
 from api.app import hash_token  # noqa: E402
 from listener.process import validate_imei_map  # noqa: E402
@@ -72,8 +75,8 @@ def main() -> int:
                     [(name, hash_token(secret)) for name, secret in tokens.items()],
                 )
                 cur.executemany(
-                    "INSERT INTO imei_map (imei, device) VALUES (%s, %s)",
-                    list(mapping.items()),
+                    "INSERT INTO imei_map (imei, meta) VALUES (%s, %s)",
+                    [(imei, Jsonb(doc)) for imei, doc in mapping.items()],
                 )
 
     print("migrated. verifying read-back...")

tools/smoke_test_api.py

@@ -5,6 +5,8 @@ Checks:
   2. API rejects requests with a wrong token (401)
   3. API accepts a valid token and returns stored messages
   4. Time framing: a wide from/to window is accepted, a reversed one is rejected
+  5. Device resources: a scratch device is upserted, read back and deleted
+  6. Token resources: a scratch token authenticates and is then removed
 
 Usage:
     python tools/smoke_test_api.py --url http://localhost:18080 --token <secret>
@@ -22,8 +24,15 @@ import urllib.request
 from urllib.parse import quote
 
 
-def request(url: str, token: str | None) -> tuple[int, str]:
-    req = urllib.request.Request(url)
+def request(
+    url: str,
+    token: str | None,
+    method: str = "GET",
+    data: bytes | None = None,
+) -> tuple[int, str]:
+    req = urllib.request.Request(url, data=data, method=method)
+    if data is not None:
+        req.add_header("Content-Type", "application/json")
     if token is not None:
         req.add_header("X-App-Token", token)
     try:
@@ -88,6 +97,75 @@ def main() -> int:
     else:
         failures.append(f"reversed from/to window -> expected 422, got {status}: {body}")
 
+    # 5. Device resources: upsert a scratch device, read it back, delete it
+    scratch_imei = "999999999999999"
+    scratch_doc = {"device": "Smoke test device", "location": "Växthus 1"}
+    status, body = request(
+        f"{args.url}/devices/{scratch_imei}",
+        token=args.token,
+        method="PUT",
+        data=json.dumps(scratch_doc).encode(),
+    )
+    if status == 200 and json.loads(body) == {"imei": scratch_imei, **scratch_doc}:
+        print("PASS: device upsert returned the saved document")
+    else:
+        failures.append(f"device upsert -> expected 200, got {status}: {body}")
+
+    status, body = request(f"{args.url}/devices/{scratch_imei}", token=args.token)
+    if status == 200 and json.loads(body) == {"imei": scratch_imei, **scratch_doc}:
+        print("PASS: device read-back matches")
+    else:
+        failures.append(f"device read-back -> expected 200, got {status}: {body}")
+
+    status, body = request(
+        f"{args.url}/devices/{scratch_imei}", token=args.token, method="DELETE"
+    )
+    if status == 200:
+        print("PASS: device deleted")
+    else:
+        failures.append(f"device delete -> expected 200, got {status}: {body}")
+
+    status, body = request(
+        f"{args.url}/devices/{scratch_imei}", token=args.token, method="DELETE"
+    )
+    if status == 404:
+        print("PASS: deleting an unknown device returns 404")
+    else:
+        failures.append(f"second device delete -> expected 404, got {status}: {body}")
+
+    # 6. Token resources: add a scratch token, authenticate with it, remove it
+    scratch_secret = "smoke-test-secret-do-not-use"
+    status, body = request(
+        f"{args.url}/tokens/smoke-test",
+        token=args.token,
+        method="PUT",
+        data=json.dumps({"token": scratch_secret}).encode(),
+    )
+    if status == 200:
+        print("PASS: token upsert")
+    else:
+        failures.append(f"token upsert -> expected 200, got {status}: {body}")
+
+    status, body = request(f"{args.url}/messages?limit=1", token=scratch_secret)
+    if status == 200:
+        print("PASS: the scratch token authenticates")
+    else:
+        failures.append(f"scratch token auth -> expected 200, got {status}: {body}")
+
+    status, body = request(
+        f"{args.url}/tokens/smoke-test", token=args.token, method="DELETE"
+    )
+    if status == 200:
+        print("PASS: token deleted")
+    else:
+        failures.append(f"token delete -> expected 200, got {status}: {body}")
+
+    status, body = request(f"{args.url}/messages?limit=1", token=scratch_secret)
+    if status == 401:
+        print("PASS: the removed token no longer authenticates")
+    else:
+        failures.append(f"removed token auth -> expected 401, got {status}: {body}")
+
     if failures:
         print("\nFAILURES:")
         for failure in failures:

tools/test_listener.py

@@ -23,9 +23,11 @@ from listener.process import (  # noqa: E402
     InjectionResult,
     parse_payload_time,
     process_message,
+    validate_imei_doc,
     validate_imei_map,
 )
 from datetime import datetime, timezone  # noqa: E402
+from typing import Any  # noqa: E402
 
 IMEI_MAP = {
     "350123456789012": "Greenhouse sensor A",
@@ -265,23 +267,58 @@ class TestPayloadTimeExtraction:
         assert result[0].payload_time is None
 
 
+class TestValidateImeiDoc:
+    """validate_imei_doc is the shared schema check for a single device
+    settings document (per-device API writes)."""
+
+    def test_accepts_document_with_extra_keys(self) -> None:
+        doc = {"device": "Device X", "location": "Växthus 1", "thresholds": {"high": 20}}
+        assert validate_imei_doc(doc, "test") == doc
+
+    def test_rejects_non_object(self) -> None:
+        with pytest.raises(ValueError, match="expected a JSON object"):
+            validate_imei_doc("Device X", "test")
+
+    def test_rejects_missing_or_bad_device(self) -> None:
+        with pytest.raises(ValueError, match="\\\"device\\\" must be a non-empty string"):
+            validate_imei_doc({"location": "Växthus 1"}, "test")
+        with pytest.raises(ValueError, match="\\\"device\\\" must be a non-empty string"):
+            validate_imei_doc({"device": 42}, "test")
+
+    def test_rejects_imei_key(self) -> None:
+        # the IMEI is the resource key / comes from the topic, never the doc
+        with pytest.raises(ValueError, match="must not contain an"):
+            validate_imei_doc({"device": "X", "imei": "350123456789012"}, "test")
+
+
 class TestValidateImeiMap:
     """validate_imei_map is the shared schema check for the imei_map table
-    writes (API request bodies and the migration tool)."""
+    writes (API config bodies and the migration tool)."""
 
-    def test_accepts_valid_map(self) -> None:
-        assert validate_imei_map(
-            {"350123456789012": "Device X"}, "test"
-        ) == {"350123456789012": "Device X"}
+    def test_accepts_document_values(self) -> None:
+        doc = {"device": "Device X", "location": "Växthus 1"}
+        assert validate_imei_map({"350123456789012": doc}, "test") == {
+            "350123456789012": doc
+        }
+
+    def test_promotes_legacy_flat_device_names(self) -> None:
+        # the old {"imei": "name"} config form is accepted and promoted
+        assert validate_imei_map({"350123456789012": "Device X"}, "test") == {
+            "350123456789012": {"device": "Device X"}
+        }
 
     def test_rejects_non_object(self) -> None:
         with pytest.raises(ValueError, match="expected a JSON object"):
             validate_imei_map([], "test")
 
-    def test_rejects_non_string_values(self) -> None:
-        with pytest.raises(ValueError, match="non-empty strings"):
+    def test_rejects_bad_documents(self) -> None:
+        with pytest.raises(ValueError, match="expected a JSON object"):
             validate_imei_map({"350123456789012": 42}, "test")
 
+    def test_rejects_empty_legacy_name(self) -> None:
+        with pytest.raises(ValueError, match="non-empty string"):
+            validate_imei_map({"350123456789012": ""}, "test")
+
     def test_rejects_non_imei_key(self) -> None:
         # Old dev EUI-style key: no longer accepted
         with pytest.raises(ValueError, match="15-digit IMEI"):
@@ -325,25 +362,51 @@ class FakeTokenStore(TokenStore):
     def names(self) -> list[str]:
         return sorted(self._tokens)
 
+    def upsert(self, app_name: str, secret: str) -> None:
+        self._tokens[app_name] = secret
+
+    def delete(self, app_name: str) -> bool:
+        if app_name in self._tokens:
+            del self._tokens[app_name]
+            return True
+        return False
+
     def replace(self, tokens: dict[str, str]) -> None:
         self.replaced_with = dict(tokens)
         self._tokens = dict(tokens)
 
 
 class FakeImeiMapStore(ImeiMapStore):
-    """In-memory imei map store: no database."""
+    """In-memory device settings store: no database."""
 
-    def __init__(self, mapping: dict[str, str]) -> None:
+    def __init__(self, mapping: dict[str, dict[str, Any]]) -> None:
         super().__init__("postgresql://unused")
-        self._mapping = dict(mapping)
-        self.replaced_with: dict[str, str] | None = None
+        self._mapping = {imei: dict(doc) for imei, doc in mapping.items()}
+        self.replaced_with: dict[str, dict[str, Any]] | None = None
+        self.upserted: tuple[str, dict[str, Any]] | None = None
+        self.deleted: str | None = None
 
-    def load(self) -> dict[str, str]:
-        return dict(self._mapping)
+    def load(self) -> dict[str, dict[str, Any]]:
+        return {imei: dict(doc) for imei, doc in self._mapping.items()}
 
-    def replace(self, mapping: dict[str, str]) -> None:
-        self.replaced_with = dict(mapping)
-        self._mapping = dict(mapping)
+    def get(self, imei: str) -> dict[str, Any] | None:
+        doc = self._mapping.get(imei)
+        return dict(doc) if doc is not None else None
+
+    def upsert(self, imei: str, doc: dict[str, Any]) -> None:
+        self.upserted = (imei, dict(doc))
+        self._mapping[imei] = dict(doc)
+
+    def delete(self, imei: str) -> bool:
+        self.deleted = imei
+        if imei in self._mapping:
+            del self._mapping[imei]
+            return True
+        return False
+
+    def replace(self, mapping: dict[str, dict[str, Any]]) -> None:
+        self.replaced_with = {imei: dict(doc) for imei, doc in mapping.items()}
+        self._mapping = {imei: dict(doc) for imei, doc in mapping.items()}
 
 
 @pytest.fixture()
@@ -372,7 +435,9 @@ def app_module(monkeypatch):
 def api(app_module):
     """ASGI app wired to in-memory fakes; yields (app, token_store, imei_store)."""
     tokens = FakeTokenStore({"test-app": "secret-abc"})
-    imeis = FakeImeiMapStore({"350123456789012": "Greenhouse sensor A"})
+    imeis = FakeImeiMapStore(
+        {"350123456789012": {"device": "Greenhouse sensor A"}}
+    )
     app = app_module.create_app(token_store=tokens, imei_store=imeis)
     return app, tokens, imeis
 
@@ -471,8 +536,8 @@ class TestConfigEndpoints:
         client, _, imeis = client_and_stores
         r = client.get("/configs/imei_map", headers=self.HDRS)
         assert r.status_code == 200
-        assert r.json() == {"350123456789012": "Greenhouse sensor A"}
-        assert imeis.load() == {"350123456789012": "Greenhouse sensor A"}
+        assert r.json() == {"350123456789012": {"device": "Greenhouse sensor A"}}
+        assert imeis.load() == {"350123456789012": {"device": "Greenhouse sensor A"}}
 
     def test_get_app_tokens_masks_secrets(self, client_and_stores) -> None:
         # secrets are stored as digests and not recoverable; only names show
@@ -493,12 +558,18 @@ class TestConfigEndpoints:
 
     def test_put_valid_imei_map(self, client_and_stores) -> None:
         client, _, imeis = client_and_stores
-        body = {"350123456789012": "Greenhouse sensor A", "490154203237518": "Water tank sensor B"}
+        body = {
+            "350123456789012": {"device": "Greenhouse sensor A", "location": "Växthus 1"},
+            "490154203237518": "Water tank sensor B",  # legacy flat form is promoted
+        }
         r = client.put("/configs/imei_map", json=body, headers=self.HDRS)
         assert r.status_code == 200
         assert r.json() == {"status": "updated", "config": "imei_map"}
-        # the store (and thereby the table) holds the new content
-        assert imeis.replaced_with == body
+        # the store (and thereby the table) holds normalized documents
+        assert imeis.replaced_with == {
+            "350123456789012": {"device": "Greenhouse sensor A", "location": "Växthus 1"},
+            "490154203237518": {"device": "Water tank sensor B"},
+        }
 
     def test_post_is_alias_for_put(self, client_and_stores) -> None:
         client, _, _ = client_and_stores
@@ -512,7 +583,8 @@ class TestConfigEndpoints:
         assert r.status_code == 422
         assert "15-digit IMEI" in r.json()["detail"]
 
-    def test_put_rejects_non_string_values(self, client_and_stores) -> None:
+    def test_put_rejects_bad_documents(self, client_and_stores) -> None:
+        # a value that is neither a settings document nor a device-name string
         client, _, _ = client_and_stores
         r = client.put("/configs/imei_map", json={"350123456789012": 42}, headers=self.HDRS)
         assert r.status_code == 422
@@ -548,4 +620,174 @@ class TestConfigEndpoints:
         # the store verifies against the replaced set immediately
         assert tokens.verify("fresh-secret")
         assert tokens.verify("secret-abc")
-        assert not tokens.verify("nope")
+        assert not tokens.verify("nope")
+
+
+class TestDeviceEndpoints:
+    """GET/PUT/DELETE /devices/{imei} — the per-device resource API a
+    management frontend uses (single-row upsert, so editing one device
+    never clobbers another)."""
+
+    HDRS = {"X-App-Token": "secret-abc"}
+
+    @pytest.fixture()
+    def client_and_store(self, api):
+        from fastapi.testclient import TestClient
+
+        app, _, imeis = api
+        return TestClient(app), imeis
+
+    def test_list_devices(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.get("/devices", headers=self.HDRS)
+        assert r.status_code == 200
+        assert r.json() == [
+            {"imei": "350123456789012", "device": "Greenhouse sensor A"}
+        ]
+
+    def test_get_device(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.get("/devices/350123456789012", headers=self.HDRS)
+        assert r.status_code == 200
+        assert r.json() == {"imei": "350123456789012", "device": "Greenhouse sensor A"}
+
+    def test_get_unknown_device_404(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.get("/devices/356938035643809", headers=self.HDRS)
+        assert r.status_code == 404
+
+    def test_get_rejects_non_imei(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.get("/devices/not-an-imei", headers=self.HDRS)
+        assert r.status_code == 422
+
+    def test_device_endpoints_require_token(self, client_and_store) -> None:
+        client, _ = client_and_store
+        assert client.get("/devices").status_code == 401
+        assert client.put("/devices/350123456789012", json={}).status_code == 401
+        assert client.delete("/devices/350123456789012").status_code == 401
+
+    def test_put_upserts_new_device(self, client_and_store) -> None:
+        client, imeis = client_and_store
+        doc = {"device": "Pump X", "location": "Växthus 1", "enabled": True}
+        r = client.put("/devices/356938035643809", json=doc, headers=self.HDRS)
+        assert r.status_code == 200
+        # the response is the saved document (+ imei), PUT-able straight back
+        assert r.json() == {"imei": "356938035643809", **doc}
+        assert imeis.get("356938035643809") == doc
+
+    def test_put_updates_existing_device_in_place(self, client_and_store) -> None:
+        # a single-device save must not wipe the other devices' rows
+        client, imeis = client_and_store
+        r = client.put(
+            "/devices/350123456789012",
+            json={"device": "Renamed"},
+            headers=self.HDRS,
+        )
+        assert r.status_code == 200
+        assert imeis.get("350123456789012") == {"device": "Renamed"}
+        # untouched: no whole-document replace() happened
+        assert imeis.replaced_with is None
+
+    def test_put_tolerates_matching_imei_key(self, client_and_store) -> None:
+        # a GET response can be edited and PUT straight back
+        client, _ = client_and_store
+        r = client.put(
+            "/devices/356938035643809",
+            json={"imei": "356938035643809", "device": "Pump X"},
+            headers=self.HDRS,
+        )
+        assert r.status_code == 200
+        assert r.json() == {"imei": "356938035643809", "device": "Pump X"}
+
+    def test_put_rejects_mismatched_imei_key(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.put(
+            "/devices/356938035643809",
+            json={"imei": "350123456789012", "device": "Pump X"},
+            headers=self.HDRS,
+        )
+        assert r.status_code == 422
+        assert "does not match" in r.json()["detail"]
+
+    def test_put_rejects_missing_device_key(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.put(
+            "/devices/356938035643809", json={"location": "X"}, headers=self.HDRS
+        )
+        assert r.status_code == 422
+
+    def test_put_rejects_non_imei_url(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.put("/devices/123", json={"device": "X"}, headers=self.HDRS)
+        assert r.status_code == 422
+
+    def test_delete_device(self, client_and_store) -> None:
+        client, imeis = client_and_store
+        r = client.delete("/devices/350123456789012", headers=self.HDRS)
+        assert r.status_code == 200
+        assert r.json() == {"status": "deleted", "imei": "350123456789012"}
+        assert imeis.get("350123456789012") is None
+
+    def test_delete_unknown_device_404(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.delete("/devices/356938035643809", headers=self.HDRS)
+        assert r.status_code == 404
+
+
+class TestTokenEndpoints:
+    """PUT/DELETE /tokens/{app} — add/rotate/remove ONE token in place."""
+
+    HDRS = {"X-App-Token": "secret-abc"}
+
+    @pytest.fixture()
+    def client_and_store(self, api):
+        from fastapi.testclient import TestClient
+
+        app, tokens, _ = api
+        return TestClient(app), tokens
+
+    def test_list_tokens_masks_secrets(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.get("/tokens", headers=self.HDRS)
+        assert r.status_code == 200
+        assert r.json() == {"tokens": {"test-app": None}}
+
+    def test_put_adds_token(self, client_and_store) -> None:
+        client, tokens = client_and_store
+        r = client.put("/tokens/frontend", json={"token": "ui-secret"}, headers=self.HDRS)
+        assert r.status_code == 200
+        assert r.json() == {"status": "updated", "app": "frontend"}
+        assert tokens.verify("ui-secret")
+
+    def test_put_rotates_token(self, client_and_store) -> None:
+        client, tokens = client_and_store
+        r = client.put("/tokens/test-app", json={"token": "rotated"}, headers=self.HDRS)
+        assert r.status_code == 200
+        assert tokens.verify("rotated")
+        assert not tokens.verify("secret-abc")
+
+    def test_put_rejects_bad_body(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.put("/tokens/frontend", json={"foo": "bar"}, headers=self.HDRS)
+        assert r.status_code == 422
+        r = client.put("/tokens/frontend", json={"token": ""}, headers=self.HDRS)
+        assert r.status_code == 422
+
+    def test_delete_token(self, client_and_store) -> None:
+        client, tokens = client_and_store
+        r = client.delete("/tokens/test-app", headers=self.HDRS)
+        assert r.status_code == 200
+        assert r.json() == {"status": "deleted", "app": "test-app"}
+        assert not tokens.verify("secret-abc")
+
+    def test_delete_unknown_token_404(self, client_and_store) -> None:
+        client, _ = client_and_store
+        r = client.delete("/tokens/does-not-exist", headers=self.HDRS)
+        assert r.status_code == 404
+
+    def test_token_endpoints_require_token(self, client_and_store) -> None:
+        client, _ = client_and_store
+        assert client.get("/tokens").status_code == 401
+        assert client.put("/tokens/x", json={"token": "y"}).status_code == 401
+        assert client.delete("/tokens/x").status_code == 401