|
|
|
@@ -0,0 +1,234 @@
|
|
|
|
|
"""FastAPI application exposing read access to the mqtt_messages table.
|
|
|
|
|
|
|
|
|
|
Authentication: static app tokens from a JSON config file, passed either as
|
|
|
|
|
the `X-App-Token` header or as an `app_token` query parameter. Tokens are
|
|
|
|
|
compared in constant time; invalid tokens yield 401.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import hmac
|
|
|
|
|
import json
|
|
|
|
|
import logging
|
|
|
|
|
import os
|
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from typing import Any
|
|
|
|
|
|
|
|
|
|
import psycopg
|
|
|
|
|
from fastapi import Depends, FastAPI, HTTPException, Query, Request, status
|
|
|
|
|
from psycopg import sql
|
|
|
|
|
|
|
|
|
|
logger = logging.getLogger("api")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class AppConfig:
|
|
|
|
|
"""Runtime configuration loaded once at startup."""
|
|
|
|
|
|
|
|
|
|
def __init__(self, dsn: str, tokens: dict[str, str], max_limit: int) -> None:
|
|
|
|
|
self.dsn = dsn
|
|
|
|
|
self.tokens = tokens
|
|
|
|
|
self.max_limit = max_limit
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_tokens(path: str) -> dict[str, str]:
|
|
|
|
|
"""Load {"tokens": {"app-name": "secret"}} from a JSON file."""
|
|
|
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
|
|
|
if not isinstance(data, dict) or not isinstance(data.get("tokens"), dict):
|
|
|
|
|
raise ValueError(f"{path}: expected JSON on the form {{\"tokens\": {{name: secret}}}}")
|
|
|
|
|
tokens = data["tokens"]
|
|
|
|
|
for name, secret in tokens.items():
|
|
|
|
|
if not isinstance(secret, str) or not secret:
|
|
|
|
|
raise ValueError(f"{path}: token {name!r} must be a non-empty string")
|
|
|
|
|
if not tokens:
|
|
|
|
|
logger.warning("%s contains no tokens; the API will reject all requests", path)
|
|
|
|
|
return dict(tokens)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _env(name: str, default: str | None = None) -> str:
|
|
|
|
|
value = os.environ.get(name)
|
|
|
|
|
if not value:
|
|
|
|
|
if default is None:
|
|
|
|
|
raise RuntimeError(f"missing required environment variable {name}")
|
|
|
|
|
return default
|
|
|
|
|
return value
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _as_utc(value: datetime) -> datetime:
|
|
|
|
|
"""Interpret naive timestamps as UTC (received_at is TIMESTAMPTZ)."""
|
|
|
|
|
return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_app() -> FastAPI:
|
|
|
|
|
app = FastAPI(
|
|
|
|
|
title="nodeledge MQTT data API",
|
|
|
|
|
version="1.0.0",
|
|
|
|
|
description="Read access to MQTT-collected JSON payloads.",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
tokens_path = _env("APP_TOKENS_FILE")
|
|
|
|
|
dsn = _env("DB_DSN")
|
|
|
|
|
max_limit = int(_env("API_MAX_LIMIT", "10000"))
|
|
|
|
|
config = AppConfig(dsn=dsn, tokens=load_tokens(tokens_path), max_limit=max_limit)
|
|
|
|
|
|
|
|
|
|
def authenticate(request: Request) -> None:
|
|
|
|
|
token = request.headers.get("X-App-Token")
|
|
|
|
|
if not token:
|
|
|
|
|
token = request.query_params.get("app_token")
|
|
|
|
|
if not token:
|
|
|
|
|
raise HTTPException(
|
|
|
|
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
|
|
|
detail="missing app token: pass the X-App-Token header",
|
|
|
|
|
)
|
|
|
|
|
for candidate in config.tokens.values():
|
|
|
|
|
if hmac.compare_digest(candidate, token):
|
|
|
|
|
return
|
|
|
|
|
raise HTTPException(
|
|
|
|
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
|
|
|
detail="invalid app token",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
app.state.config = config
|
|
|
|
|
app.state.authenticate = authenticate
|
|
|
|
|
|
|
|
|
|
@app.get("/health", tags=["meta"])
|
|
|
|
|
def health() -> dict[str, Any]:
|
|
|
|
|
return {"status": "ok"}
|
|
|
|
|
|
|
|
|
|
@app.get("/messages/count", tags=["meta"], dependencies=[Depends(authenticate)])
|
|
|
|
|
def message_count() -> dict[str, int]:
|
|
|
|
|
conn = psycopg.connect(config.dsn)
|
|
|
|
|
try:
|
|
|
|
|
with conn.cursor() as cur:
|
|
|
|
|
cur.execute("SELECT count(*) FROM mqtt_messages")
|
|
|
|
|
row = cur.fetchone()
|
|
|
|
|
finally:
|
|
|
|
|
conn.close()
|
|
|
|
|
count = row[0] if row is not None else 0
|
|
|
|
|
return {"count": count}
|
|
|
|
|
|
|
|
|
|
@app.get("/messages", tags=["data"], dependencies=[Depends(authenticate)])
|
|
|
|
|
def list_messages(
|
|
|
|
|
imei: str | None = Query(None, description="Filter by IMEI (15 digits)"),
|
|
|
|
|
device: str | None = Query(None, description="Filter by device name"),
|
|
|
|
|
from_time: datetime | None = Query(
|
|
|
|
|
None,
|
|
|
|
|
alias="from",
|
|
|
|
|
description=(
|
|
|
|
|
"Only messages received at or after this time (ISO 8601, inclusive; "
|
|
|
|
|
"naive values are treated as UTC)"
|
|
|
|
|
),
|
|
|
|
|
),
|
|
|
|
|
to_time: datetime | None = Query(
|
|
|
|
|
None,
|
|
|
|
|
alias="to",
|
|
|
|
|
description=(
|
|
|
|
|
"Only messages received at or before this time (ISO 8601, inclusive; "
|
|
|
|
|
"naive values are treated as UTC)"
|
|
|
|
|
),
|
|
|
|
|
),
|
|
|
|
|
payload_from_time: datetime | None = Query(
|
|
|
|
|
None,
|
|
|
|
|
alias="payload_from",
|
|
|
|
|
description=(
|
|
|
|
|
"Frame on the device timestamp extracted from the JSON payload "
|
|
|
|
|
"(ISO 8601, inclusive; naive values are treated as UTC). Messages "
|
|
|
|
|
"without a parseable payload timestamp are excluded."
|
|
|
|
|
),
|
|
|
|
|
),
|
|
|
|
|
payload_to_time: datetime | None = Query(
|
|
|
|
|
None,
|
|
|
|
|
alias="payload_to",
|
|
|
|
|
description=(
|
|
|
|
|
"Frame on the device timestamp extracted from the JSON payload "
|
|
|
|
|
"(ISO 8601, inclusive; naive values are treated as UTC). Messages "
|
|
|
|
|
"without a parseable payload timestamp are excluded."
|
|
|
|
|
),
|
|
|
|
|
),
|
|
|
|
|
limit: int = Query(100, gt=0, description="Page size"),
|
|
|
|
|
offset: int = Query(0, ge=0, description="Page offset"),
|
|
|
|
|
) -> dict[str, Any]:
|
|
|
|
|
if from_time is not None and to_time is not None and from_time > to_time:
|
|
|
|
|
raise HTTPException(
|
|
|
|
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
|
|
|
detail="'from' must be earlier than or equal to 'to'",
|
|
|
|
|
)
|
|
|
|
|
if (
|
|
|
|
|
payload_from_time is not None
|
|
|
|
|
and payload_to_time is not None
|
|
|
|
|
and payload_from_time > payload_to_time
|
|
|
|
|
):
|
|
|
|
|
raise HTTPException(
|
|
|
|
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
|
|
|
detail="'payload_from' must be earlier than or equal to 'payload_to'",
|
|
|
|
|
)
|
|
|
|
|
limit = min(limit, config.max_limit)
|
|
|
|
|
conditions: list[sql.Composable] = []
|
|
|
|
|
params: list[Any] = []
|
|
|
|
|
if imei:
|
|
|
|
|
conditions.append(sql.SQL("imei = %s"))
|
|
|
|
|
params.append(imei)
|
|
|
|
|
if device:
|
|
|
|
|
conditions.append(sql.SQL("device = %s"))
|
|
|
|
|
params.append(device)
|
|
|
|
|
if from_time is not None:
|
|
|
|
|
conditions.append(sql.SQL("received_at >= %s"))
|
|
|
|
|
params.append(_as_utc(from_time))
|
|
|
|
|
if to_time is not None:
|
|
|
|
|
conditions.append(sql.SQL("received_at <= %s"))
|
|
|
|
|
params.append(_as_utc(to_time))
|
|
|
|
|
if payload_from_time is not None:
|
|
|
|
|
conditions.append(sql.SQL("payload_time >= %s"))
|
|
|
|
|
params.append(_as_utc(payload_from_time))
|
|
|
|
|
if payload_to_time is not None:
|
|
|
|
|
conditions.append(sql.SQL("payload_time <= %s"))
|
|
|
|
|
params.append(_as_utc(payload_to_time))
|
|
|
|
|
|
|
|
|
|
where = sql.SQL("")
|
|
|
|
|
if conditions:
|
|
|
|
|
where = sql.SQL(" WHERE ") + sql.SQL(" AND ").join(conditions)
|
|
|
|
|
|
|
|
|
|
# Order by device timestamp when framing on it (NULLS LAST matches the
|
|
|
|
|
# idx_mqtt_messages_ptime index); otherwise newest-first by insert id.
|
|
|
|
|
if payload_from_time is not None or payload_to_time is not None:
|
|
|
|
|
order_by = sql.SQL(" ORDER BY payload_time DESC NULLS LAST LIMIT %s OFFSET %s")
|
|
|
|
|
else:
|
|
|
|
|
order_by = sql.SQL(" ORDER BY id DESC LIMIT %s OFFSET %s")
|
|
|
|
|
|
|
|
|
|
query = sql.SQL(
|
|
|
|
|
"SELECT id, received_at, topic, imei, device, payload, payload_time FROM mqtt_messages"
|
|
|
|
|
) + where + order_by
|
|
|
|
|
|
|
|
|
|
conn = psycopg.connect(config.dsn)
|
|
|
|
|
try:
|
|
|
|
|
with conn.cursor() as cur:
|
|
|
|
|
cur.execute(query, (*params, limit, offset))
|
|
|
|
|
rows = cur.fetchall()
|
|
|
|
|
finally:
|
|
|
|
|
conn.close()
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
"limit": limit,
|
|
|
|
|
"offset": offset,
|
|
|
|
|
"from": from_time.isoformat() if from_time is not None else None,
|
|
|
|
|
"to": to_time.isoformat() if to_time is not None else None,
|
|
|
|
|
"payload_from": payload_from_time.isoformat() if payload_from_time is not None else None,
|
|
|
|
|
"payload_to": payload_to_time.isoformat() if payload_to_time is not None else None,
|
|
|
|
|
"items": [
|
|
|
|
|
{
|
|
|
|
|
"id": row[0],
|
|
|
|
|
"received_at": row[1].isoformat(),
|
|
|
|
|
"topic": row[2],
|
|
|
|
|
"imei": row[3],
|
|
|
|
|
"device": row[4],
|
|
|
|
|
"payload": row[5],
|
|
|
|
|
"payload_time": row[6].isoformat() if row[6] is not None else None,
|
|
|
|
|
}
|
|
|
|
|
for row in rows
|
|
|
|
|
],
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return app
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app = create_app()
|
Jonas Bengtsson