🚧 Add sqlite extraction
This commit is contained in:
Jonas Bengtsson
@@ -0,0 +1 @@
|
||||
"""Content extraction into derived artifacts (Section 45)."""
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Artifact extraction framework (Section 45).
|
||||
|
||||
Derived artifacts are plain-text files generated from binary sources (e.g.
|
||||
SQLite databases) and versioned in the repository so Git history shows
|
||||
readable, row-level changes instead of opaque binary blobs.
|
||||
|
||||
Follows the same pluggable-interface principle as author resolution
|
||||
(Section 44): an ArtifactExtractor abstraction with format-specific
|
||||
implementations added independently.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExtractionSource:
|
||||
"""One configured binary source (Section 45.3)."""
|
||||
|
||||
#: Repository-relative path of the source database (POSIX separators)
|
||||
path: str
|
||||
#: Tables to extract; empty means all user tables
|
||||
tables: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionResult:
|
||||
"""Outcome of extracting one source database."""
|
||||
|
||||
source: str
|
||||
#: Repository-relative artifact paths written or refreshed
|
||||
artifacts: list[str] = field(default_factory=list)
|
||||
#: Repository-relative artifact paths removed (stale, Section 45.6)
|
||||
removed: list[str] = field(default_factory=list)
|
||||
#: True when nothing changed relative to the previous artifacts
|
||||
unchanged: bool = False
|
||||
|
||||
|
||||
class ArtifactExtractor(ABC):
|
||||
"""Pluggable content extractor (Section 45.1).
|
||||
|
||||
Implementations refresh derived artifacts from one configured source
|
||||
into a sub-folder of the output root named after the source file.
|
||||
|
||||
Contract (Sections 45.4 - 45.7):
|
||||
* open the source read-only; never write to it (FR-034);
|
||||
* operate on a snapshot when the live source may change concurrently;
|
||||
* produce byte-identical output for unchanged content (FR-031);
|
||||
* remove artifacts whose source table no longer exists (FR-033);
|
||||
* raise ExtractionError when the source cannot be read or an
|
||||
artifact cannot be written; the caller aborts the collection
|
||||
(FR-032) - silent staleness is not acceptable.
|
||||
"""
|
||||
|
||||
#: Registry name used in configuration (extraction.sources[].format)
|
||||
name: str = "abstract"
|
||||
|
||||
@abstractmethod
|
||||
def extract(
|
||||
self,
|
||||
source: ExtractionSource,
|
||||
repository_root: Path,
|
||||
output_root: Path,
|
||||
) -> ExtractionResult:
|
||||
"""Refresh the derived artifacts for one source; returns its result."""
|
||||
|
||||
|
||||
class ExtractionError(Exception):
|
||||
"""Raised when extraction fails; aborts the collection (FR-032)."""
|
||||
|
||||
|
||||
def default_output_root(name: str) -> str:
|
||||
"""Repository-relative output root folder (Section 45.2, default)."""
|
||||
return name if name else "extracted"
|
||||
|
||||
|
||||
def artifact_folder(source: ExtractionSource) -> str:
|
||||
"""Sub-folder name for one source database (Section 45.2).
|
||||
|
||||
Named after the database file (without extension), so table artifact
|
||||
files cannot collide between databases.
|
||||
"""
|
||||
stem = Path(source.path).stem
|
||||
return stem or "source"
|
||||
|
||||
|
||||
def artifact_filename(table: str) -> str:
|
||||
"""Deterministic, platform-safe artifact file name for a table (FR-029)."""
|
||||
# Keep characters that are safe on Windows/Linux/macOS; replace the
|
||||
# rest with underscores; collapse runs; strip leading/trailing ones.
|
||||
safe = "".join(ch if (ch.isalnum() or ch in "._- ") else "_" for ch in table)
|
||||
while "__" in safe:
|
||||
safe = safe.replace("__", "_")
|
||||
safe = safe.strip("._")
|
||||
reserved = {
|
||||
"con", "prn", "aux", "nul",
|
||||
"com1", "com2", "com3", "com4", "com5", "com6", "com7", "com8", "com9",
|
||||
"lpt1", "lpt2", "lpt3", "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9",
|
||||
}
|
||||
if safe.lower() in reserved:
|
||||
safe = f"_{safe}"
|
||||
return f"{safe or 'table'}.jsonl"
|
||||
|
||||
|
||||
def to_jsonl(rows: list[dict[str, object]]) -> str:
|
||||
"""Serialize rows to deterministic JSON Lines (Section 45.4).
|
||||
|
||||
Keys sorted for stable column order; ensure_ascii=False for readable,
|
||||
byte-stable UTF-8; separators without padding for minimal diffs.
|
||||
Rows must already be in deterministic order (caller orders by PK).
|
||||
"""
|
||||
import json
|
||||
|
||||
lines = [
|
||||
json.dumps(row, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
for row in rows
|
||||
]
|
||||
return "\n".join(lines) + "\n" if lines else ""
|
||||
|
||||
|
||||
def write_if_changed(path: Path, content: str) -> bool:
|
||||
"""Write content only when it differs; returns True when written.
|
||||
|
||||
Avoiding unconditional writes keeps file mtime (and therefore the
|
||||
commit message's last-modified column) stable for unchanged tables,
|
||||
and prevents spurious change records.
|
||||
"""
|
||||
if path.exists():
|
||||
try:
|
||||
if path.read_text(encoding="utf-8") == content:
|
||||
return False
|
||||
except OSError:
|
||||
pass
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8", newline="\n")
|
||||
return True
|
||||
|
||||
|
||||
def remove_if_exists(path: Path) -> bool:
|
||||
"""Delete a stale artifact file; returns True when removed (FR-033)."""
|
||||
if not path.exists():
|
||||
return False
|
||||
try:
|
||||
path.unlink()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def timestamp_note(moment: dt.datetime) -> str: # pragma: no cover - future use
|
||||
"""Reserved for potential sidecar metadata (Section 45.9)."""
|
||||
return moment.isoformat()
|
||||
@@ -0,0 +1,202 @@
|
||||
"""SQLite artifact extractor (Section 45).
|
||||
|
||||
Extracts configured tables of a SQLite database into JSON Lines artifacts
|
||||
under the output root:
|
||||
|
||||
<output_root>/<database-stem>/<table>.jsonl
|
||||
|
||||
Requirements implemented here (Section 45.4 - 45.6):
|
||||
* read-only source access via snapshot (FR-034): the database file and its
|
||||
WAL sidecar are copied to a temporary location before reading, so a
|
||||
live SCADA system writing in WAL mode is never disturbed and a
|
||||
partially written transaction is never captured;
|
||||
* deterministic output (FR-031): rows ordered by primary key (rowid
|
||||
fallback), stable JSON encoding, no run-dependent values;
|
||||
* stale artifact removal (FR-033): artifact files for tables that no
|
||||
longer exist in the database are deleted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from git_auto_tracker.errors import CollectionError
|
||||
from git_auto_tracker.extraction.base import (
|
||||
ExtractionError,
|
||||
ExtractionResult,
|
||||
ExtractionSource,
|
||||
artifact_filename,
|
||||
artifact_folder,
|
||||
remove_if_exists,
|
||||
to_jsonl,
|
||||
write_if_changed,
|
||||
)
|
||||
|
||||
_USER_TABLES_SQL = (
|
||||
"SELECT name FROM sqlite_master "
|
||||
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%' "
|
||||
"ORDER BY name"
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _snapshot_database(source_path: Path):
|
||||
"""Copy the database (and WAL sidecar) to temp; open the copy (45.5).
|
||||
|
||||
Opening the snapshot instead of the live file guarantees read-only
|
||||
access to the source (FR-034) and a consistent view even when the
|
||||
source is being written concurrently (Section 20). The connection is
|
||||
closed before the temporary directory is removed; otherwise Windows
|
||||
keeps the file locked and cleanup fails.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory(prefix="git-auto-tracker-extract-") as tmp:
|
||||
snapshot = Path(tmp) / source_path.name
|
||||
shutil.copy2(source_path, snapshot)
|
||||
for sidecar in ("-wal", "-shm"):
|
||||
live = source_path.with_name(source_path.name + sidecar)
|
||||
if live.exists():
|
||||
shutil.copy2(live, snapshot.with_name(snapshot.name + sidecar))
|
||||
connection = sqlite3.connect(f"file:{snapshot}?mode=ro", uri=True)
|
||||
try:
|
||||
yield connection
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _order_by(table: str, columns: list[str]) -> str:
|
||||
"""Deterministic ORDER BY clause: primary key columns, rowid fallback."""
|
||||
pk_columns = [c for c in columns if not c.startswith("_rowid_")]
|
||||
quoted = ", ".join(f'"{c}"' for c in pk_columns) if pk_columns else "rowid"
|
||||
return quoted
|
||||
|
||||
|
||||
def _primary_key_columns(cursor: sqlite3.Cursor, table: str) -> list[str]:
|
||||
"""Primary key column names for a table, in index order; [] if none."""
|
||||
pk_columns: list[str] = []
|
||||
for row in cursor.execute(f'PRAGMA table_info("{table}")'):
|
||||
# row: (cid, name, type, notnull, dflt_value, pk)
|
||||
if row[5]: # pk index (1-based ordinal)
|
||||
while len(pk_columns) < row[5]:
|
||||
pk_columns.append("")
|
||||
pk_columns[row[5] - 1] = str(row[1])
|
||||
return pk_columns
|
||||
|
||||
|
||||
def _sanitize(identifier: str) -> str:
|
||||
"""Quote an identifier for SQL; rejects embedded quotes (FR-029)."""
|
||||
if '"' in identifier:
|
||||
raise ExtractionError(f"invalid table name: {identifier!r}")
|
||||
return f'"{identifier}"'
|
||||
|
||||
|
||||
class SqliteExtractor:
|
||||
"""Extracts SQLite tables to JSON Lines artifacts (Section 45)."""
|
||||
|
||||
name = "sqlite"
|
||||
|
||||
def extract(
|
||||
self,
|
||||
source: ExtractionSource,
|
||||
repository_root: Path,
|
||||
output_root: Path,
|
||||
) -> ExtractionResult:
|
||||
source_path = repository_root / Path(source.path)
|
||||
if not source_path.is_file():
|
||||
raise ExtractionError(f"source database not found: {source.path}")
|
||||
|
||||
folder = output_root / artifact_folder(source)
|
||||
result = ExtractionResult(source=source.path)
|
||||
|
||||
try:
|
||||
with _snapshot_database(source_path) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
cursor = connection.cursor()
|
||||
tables = self._resolve_tables(cursor, source.tables)
|
||||
for table in tables:
|
||||
artifact = folder / artifact_filename(table)
|
||||
rel = self._relative(repository_root, artifact)
|
||||
content = self._dump_table(cursor, table)
|
||||
if write_if_changed(artifact, content):
|
||||
result.artifacts.append(rel)
|
||||
else:
|
||||
result.unchanged = True
|
||||
result.removed = self._remove_stale(folder, tables, repository_root)
|
||||
except ExtractionError:
|
||||
raise
|
||||
except sqlite3.Error as exc:
|
||||
raise ExtractionError(f"failed to extract {source.path}: {exc}") from exc
|
||||
except OSError as exc:
|
||||
raise ExtractionError(f"cannot read source {source.path}: {exc}") from exc
|
||||
|
||||
if not result.artifacts and not result.removed and not result.unchanged:
|
||||
# Source had no user tables at all
|
||||
result.unchanged = True
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _relative(repository_root: Path, path: Path) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(repository_root.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return path.as_posix()
|
||||
|
||||
def _resolve_tables(self, cursor: sqlite3.Cursor, wanted: tuple[str, ...]) -> list[str]:
|
||||
"""Configured tables validated against the database; empty = all."""
|
||||
available = [
|
||||
str(row[0])
|
||||
for row in cursor.execute(_USER_TABLES_SQL)
|
||||
]
|
||||
if not wanted:
|
||||
return available
|
||||
known = set(available)
|
||||
for table in wanted:
|
||||
if table not in known:
|
||||
raise ExtractionError(
|
||||
f"configured table {table!r} does not exist; "
|
||||
f"available: {', '.join(available) or 'none'}"
|
||||
)
|
||||
_sanitize(table)
|
||||
return [t for t in available if t in set(wanted)]
|
||||
|
||||
def _dump_table(self, cursor: sqlite3.Cursor, table: str) -> str:
|
||||
"""One JSON object per row, ordered by primary key (Section 45.4)."""
|
||||
pk = _primary_key_columns(cursor, table)
|
||||
order = ", ".join(f'"{c}"' for c in pk) if pk else "rowid"
|
||||
rows = [
|
||||
{str(key): value for key, value in dict(row).items()}
|
||||
for row in cursor.execute(
|
||||
f"SELECT * FROM {_sanitize(table)} ORDER BY {order}"
|
||||
)
|
||||
]
|
||||
return to_jsonl(rows)
|
||||
|
||||
def _remove_stale(
|
||||
self,
|
||||
folder: Path,
|
||||
current_tables: list[str],
|
||||
repository_root: Path,
|
||||
) -> list[str]:
|
||||
"""Delete artifact files whose table no longer exists (FR-033)."""
|
||||
expected = {artifact_filename(table) for table in current_tables}
|
||||
removed: list[str] = []
|
||||
if not folder.is_dir():
|
||||
return removed
|
||||
for existing in sorted(folder.iterdir()):
|
||||
if existing.is_file() and existing.name not in expected and remove_if_exists(existing):
|
||||
removed.append(self._relative(repository_root, existing))
|
||||
return removed
|
||||
|
||||
|
||||
def extractor_for(name: str) -> SqliteExtractor:
|
||||
"""Registry lookup for configured extractor names (Section 45.3)."""
|
||||
registry = {SqliteExtractor.name: SqliteExtractor}
|
||||
factory = registry.get(name)
|
||||
if factory is None:
|
||||
raise CollectionError(
|
||||
f"unknown extraction source format: {name!r}; available: {list(registry)}"
|
||||
)
|
||||
return factory()
|
||||
Reference in New Issue
Block a user