🚧 Add sqlite extraction
This commit is contained in:
Jonas Bengtsson
@@ -169,6 +169,132 @@ class TestRunCycle:
|
||||
assert "Station PS101/räumliche Datei.xml" in files
|
||||
|
||||
|
||||
class TestExtraction:
|
||||
"""Section 45: derived artifacts flow through the full collection."""
|
||||
|
||||
def _make_sqlite(self, git_repo: Path) -> Path:
|
||||
import sqlite3
|
||||
|
||||
history = git_repo / "history"
|
||||
history.mkdir(exist_ok=True)
|
||||
db = history / "alarms.sqlite"
|
||||
connection = sqlite3.connect(db)
|
||||
connection.execute("CREATE TABLE alarms (id INTEGER PRIMARY KEY, text TEXT)")
|
||||
connection.execute("INSERT INTO alarms VALUES (1, 'alpha')")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
return db
|
||||
|
||||
def _config_with_extraction(self, tracker_config: dict) -> dict:
|
||||
tracker_config["extraction"] = {
|
||||
"enabled": True,
|
||||
"output_root": "extracted",
|
||||
"format": "jsonl",
|
||||
"sources": [{"path": "history/alarms.sqlite"}],
|
||||
}
|
||||
tracker_config["files"]["exclude"].append("**/*.sqlite")
|
||||
return tracker_config
|
||||
|
||||
def test_first_run_commits_artifacts(
|
||||
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||
) -> None:
|
||||
self._make_sqlite(git_repo)
|
||||
config = self._config_with_extraction(tracker_config)
|
||||
outcome = _run(config, tmp_path)
|
||||
# artifacts only: raw DB excluded, no other changes
|
||||
assert outcome.change_count == 1
|
||||
assert outcome.commit_hash is not None
|
||||
body = _git_out("log", "-1", "--format=%b", cwd=git_repo)
|
||||
assert "extracted/alarms/alarms.jsonl" in body
|
||||
tracked = _git_out("ls-files", "-z", cwd=git_repo)
|
||||
assert "alarms.sqlite" not in tracked # raw DB excluded
|
||||
content = (git_repo / "extracted" / "alarms" / "alarms.jsonl").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert '{"id":1,"text":"alpha"}' in content
|
||||
|
||||
def test_unchanged_db_no_second_commit(
|
||||
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||
) -> None:
|
||||
self._make_sqlite(git_repo)
|
||||
config = self._config_with_extraction(tracker_config)
|
||||
_run(config, tmp_path)
|
||||
outcome = _run(config, tmp_path)
|
||||
assert outcome.change_count == 0 # FR-031: byte-identical artifacts
|
||||
assert outcome.commit_hash is None
|
||||
|
||||
def test_changed_row_produces_artifact_commit(
|
||||
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||
) -> None:
|
||||
import sqlite3
|
||||
|
||||
db = self._make_sqlite(git_repo)
|
||||
config = self._config_with_extraction(tracker_config)
|
||||
_run(config, tmp_path)
|
||||
|
||||
connection = sqlite3.connect(db)
|
||||
connection.execute("INSERT INTO alarms VALUES (2, 'beta')")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
outcome = _run(config, tmp_path)
|
||||
assert outcome.change_count == 1
|
||||
body = _git_out("log", "-1", "--format=%b", cwd=git_repo)
|
||||
assert "M ; extracted/alarms/alarms.jsonl" in body
|
||||
|
||||
def test_failed_source_skipped_with_warning_others_extracted(
|
||||
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||
) -> None:
|
||||
"""Section 45.7: per-source isolation - one bad file, others proceed."""
|
||||
self._make_sqlite(git_repo)
|
||||
# A second, non-database source alongside the valid one
|
||||
(git_repo / "history" / "impostor.db").write_text(
|
||||
"definitely not a database", encoding="utf-8"
|
||||
)
|
||||
config = self._config_with_extraction(tracker_config)
|
||||
config["extraction"]["sources"].append({"path": "history/impostor.db"})
|
||||
config["files"]["exclude"].append("history/impostor.db") # raw file not tracked
|
||||
|
||||
outcome = _run(config, tmp_path)
|
||||
|
||||
# The valid source was still extracted and committed
|
||||
assert outcome.change_count == 1
|
||||
assert outcome.commit_hash is not None
|
||||
body = _git_out("log", "-1", "--format=%b", cwd=git_repo)
|
||||
assert "extracted/alarms/alarms.jsonl" in body
|
||||
# Nothing was extracted for the bad source
|
||||
assert not (git_repo / "extracted" / "impostor").exists()
|
||||
|
||||
def test_non_sqlite_file_not_committed_as_artifact(
|
||||
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||
) -> None:
|
||||
"""A non-database source is skipped, never dumped or committed."""
|
||||
history = git_repo / "history"
|
||||
history.mkdir(exist_ok=True)
|
||||
(git_repo / "history" / "notadb.sqlite").write_text(
|
||||
"plain text wearing a database name", encoding="utf-8"
|
||||
)
|
||||
config = self._config_with_extraction(tracker_config)
|
||||
config["extraction"]["sources"] = [{"path": "history/notadb.sqlite"}]
|
||||
|
||||
outcome = _run(config, tmp_path)
|
||||
assert outcome.change_count == 0 # nothing extracted, nothing committed
|
||||
assert not (git_repo / "extracted" / "notadb").exists()
|
||||
|
||||
def test_dry_run_skips_refresh(
|
||||
self, git_repo: Path, tracker_config: dict, tmp_path: Path, caplog
|
||||
) -> None:
|
||||
self._make_sqlite(git_repo)
|
||||
config = self._config_with_extraction(tracker_config)
|
||||
before = _git_out("status", "--porcelain", "-z", cwd=git_repo)
|
||||
outcome = _run(config, tmp_path, dry_run=True)
|
||||
assert outcome.dry_run
|
||||
# Artifacts never created: the refresh is the step that writes them
|
||||
assert (git_repo / "extracted").exists() is False # tree untouched
|
||||
after = _git_out("status", "--porcelain", "-z", cwd=git_repo)
|
||||
assert before == after
|
||||
|
||||
|
||||
class TestSafetyAborts:
|
||||
def test_pre_existing_staged_changes_abort(
|
||||
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Unit tests for the SQLite artifact extractor (Section 45).
|
||||
|
||||
Covers the Section 41 extraction test list: artifacts produced per
|
||||
table, byte-identical output for unchanged content, deterministic
|
||||
ordering and serialization, stale removal, failure policy, and
|
||||
read-only source access.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from git_auto_tracker.errors import CollectionError
|
||||
from git_auto_tracker.extraction.base import (
|
||||
ExtractionError,
|
||||
ExtractionSource,
|
||||
artifact_filename,
|
||||
artifact_folder,
|
||||
to_jsonl,
|
||||
write_if_changed,
|
||||
)
|
||||
from git_auto_tracker.extraction.sqlite import SqliteExtractor, extractor_for
|
||||
|
||||
|
||||
def _make_db(path: Path, tables: dict[str, list[tuple[str, ...]]]) -> None:
|
||||
"""Create the fixture database; idempotent for repeat extraction runs."""
|
||||
connection = sqlite3.connect(path)
|
||||
try:
|
||||
for ddl in (
|
||||
"CREATE TABLE IF NOT EXISTS alarms (id INTEGER PRIMARY KEY, text TEXT)",
|
||||
"CREATE TABLE IF NOT EXISTS events "
|
||||
"(id INTEGER PRIMARY KEY, source TEXT, value REAL)",
|
||||
):
|
||||
if ddl.split()[5] in tables:
|
||||
connection.execute(ddl)
|
||||
for table, rows in tables.items():
|
||||
for row in rows:
|
||||
connection.execute(
|
||||
f'INSERT OR IGNORE INTO "{table}" VALUES ({",".join("?" * len(row))})',
|
||||
row,
|
||||
)
|
||||
connection.commit()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _read(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
class TestHelpers:
|
||||
def test_artifact_folder_named_after_database(self) -> None:
|
||||
source = ExtractionSource(path="history/alarms.sqlite")
|
||||
assert artifact_folder(source) == "alarms"
|
||||
|
||||
def test_artifact_filename_sanitized(self) -> None:
|
||||
assert artifact_filename("alarms") == "alarms.jsonl"
|
||||
assert artifact_filename("weird name") == "weird name.jsonl"
|
||||
assert artifact_filename('bad"quote') == 'bad_quote.jsonl'
|
||||
assert artifact_filename("com1") == "_com1.jsonl"
|
||||
assert artifact_filename("") == "table.jsonl"
|
||||
|
||||
def test_to_jsonl_sorted_and_stable(self) -> None:
|
||||
rows = [{"b": 1, "a": 2}, {"a": 3, "b": 4}]
|
||||
assert to_jsonl(rows) == '{"a":2,"b":1}\n{"a":3,"b":4}\n'
|
||||
|
||||
def test_to_jsonl_empty(self) -> None:
|
||||
assert to_jsonl([]) == ""
|
||||
|
||||
def test_write_if_changed_skips_identical(self, tmp_path: Path) -> None:
|
||||
target = tmp_path / "x.jsonl"
|
||||
assert write_if_changed(target, "a\n") is True
|
||||
assert write_if_changed(target, "a\n") is False # identical, no write
|
||||
assert write_if_changed(target, "b\n") is True
|
||||
|
||||
def test_extractor_for_unknown_name(self) -> None:
|
||||
with pytest.raises(CollectionError, match="unknown extraction"):
|
||||
extractor_for("csv")
|
||||
|
||||
|
||||
class TestSqliteExtractor:
|
||||
def _extract(self, tmp_path: Path, tables: tuple[str, ...] = ()) -> object:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir(exist_ok=True)
|
||||
db = repo / "history"
|
||||
db.mkdir(exist_ok=True)
|
||||
_make_db(
|
||||
repo / "history" / "alarms.sqlite",
|
||||
{
|
||||
"alarms": [(2, "beta"), (1, "alpha")],
|
||||
"events": [(1, "s1", 1.5), (2, "s2", 2.5)],
|
||||
},
|
||||
)
|
||||
source = ExtractionSource(path="history/alarms.sqlite", tables=tables)
|
||||
extractor = SqliteExtractor()
|
||||
return extractor.extract(source, repo, repo / "extracted")
|
||||
|
||||
def test_all_user_tables_extracted(self, tmp_path: Path) -> None:
|
||||
result = self._extract(tmp_path)
|
||||
repo = tmp_path / "repo"
|
||||
assert "extracted/alarms/alarms.jsonl" in result.artifacts
|
||||
assert "extracted/alarms/events.jsonl" in result.artifacts
|
||||
content = _read(repo / "extracted" / "alarms" / "alarms.jsonl")
|
||||
# Ordered by primary key: 1 before 2 regardless of insertion order
|
||||
assert content == '{"id":1,"text":"alpha"}\n{"id":2,"text":"beta"}\n'
|
||||
|
||||
def test_selected_tables_only(self, tmp_path: Path) -> None:
|
||||
result = self._extract(tmp_path, tables=("alarms",))
|
||||
assert "extracted/alarms/alarms.jsonl" in result.artifacts
|
||||
assert not (tmp_path / "repo" / "extracted" / "alarms" / "events.jsonl").exists()
|
||||
|
||||
def test_unknown_table_rejected(self, tmp_path: Path) -> None:
|
||||
with pytest.raises(ExtractionError, match="does not exist"):
|
||||
self._extract(tmp_path, tables=("nope",))
|
||||
|
||||
def test_byte_identical_on_re_extraction(self, tmp_path: Path) -> None:
|
||||
self._extract(tmp_path)
|
||||
repo = tmp_path / "repo"
|
||||
artifact = repo / "extracted" / "alarms" / "alarms.jsonl"
|
||||
first = _read(artifact)
|
||||
result2 = self._extract(tmp_path)
|
||||
assert _read(artifact) == first # FR-031 byte-stable
|
||||
assert result2.artifacts == [] # nothing rewritten
|
||||
assert result2.unchanged is True
|
||||
|
||||
def test_changed_row_updates_artifact(self, tmp_path: Path) -> None:
|
||||
self._extract(tmp_path)
|
||||
repo = tmp_path / "repo"
|
||||
connection = sqlite3.connect(repo / "history" / "alarms.sqlite")
|
||||
connection.execute("INSERT INTO alarms VALUES (3, 'gamma')")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
result = self._extract(tmp_path)
|
||||
assert "extracted/alarms/alarms.jsonl" in result.artifacts
|
||||
assert '{"id":3,"text":"gamma"}' in _read(
|
||||
repo / "extracted" / "alarms" / "alarms.jsonl"
|
||||
)
|
||||
|
||||
def test_stale_table_artifact_removed(self, tmp_path: Path) -> None:
|
||||
self._extract(tmp_path)
|
||||
repo = tmp_path / "repo"
|
||||
connection = sqlite3.connect(repo / "history" / "alarms.sqlite")
|
||||
connection.execute("DROP TABLE events")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
result = self._extract(tmp_path, tables=("alarms",))
|
||||
# events artifact from the earlier run has no source table anymore
|
||||
assert "extracted/alarms/events.jsonl" in result.removed
|
||||
assert not (repo / "extracted" / "alarms" / "events.jsonl").exists()
|
||||
|
||||
def test_missing_source_raises(self, tmp_path: Path) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir(exist_ok=True)
|
||||
source = ExtractionSource(path="history/absent.sqlite")
|
||||
with pytest.raises(ExtractionError, match="not found"):
|
||||
SqliteExtractor().extract(source, repo, repo / "extracted")
|
||||
|
||||
def test_source_never_modified(self, tmp_path: Path) -> None:
|
||||
"""FR-034: extraction must not change the source database file."""
|
||||
result = self._extract(tmp_path)
|
||||
db = tmp_path / "repo" / "history" / "alarms.sqlite"
|
||||
connection = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||
rows = list(connection.execute("SELECT COUNT(*) FROM alarms"))
|
||||
connection.close()
|
||||
assert rows[0][0] == 2
|
||||
_ = result
|
||||
|
||||
def test_wal_database_extracted_consistently(self, tmp_path: Path) -> None:
|
||||
"""Section 45.5: a WAL-mode source is snapshotted, not locked."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir(exist_ok=True)
|
||||
(repo / "history").mkdir(exist_ok=True)
|
||||
db = repo / "history" / "wal.sqlite"
|
||||
connection = sqlite3.connect(db)
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
connection.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
|
||||
connection.execute("INSERT INTO t VALUES (1, 'x')")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
source = ExtractionSource(path="history/wal.sqlite")
|
||||
result = SqliteExtractor().extract(source, repo, repo / "extracted")
|
||||
assert "extracted/wal/t.jsonl" in result.artifacts
|
||||
Reference in New Issue
Block a user