🚧 Add export feature and refined exclusion info
This commit is contained in:
@@ -14,7 +14,7 @@ See `functional specification.md` for the full specification.
|
|||||||
|
|
||||||
- Run-once execution, designed for Windows Task Scheduler / cron / systemd timers
|
- Run-once execution, designed for Windows Task Scheduler / cron / systemd timers
|
||||||
- Change detection via `git status --porcelain -z` (NUL-safe, UTF-8)
|
- Change detection via `git status --porcelain -z` (NUL-safe, UTF-8)
|
||||||
- Configurable include/exclude file filters (globs)
|
- Configurable include/exclude file filters (globs), with every excluded file and its matching rule logged at DEBUG level
|
||||||
- Security validation with forbidden patterns (abort by default)
|
- Security validation with forbidden patterns (abort by default)
|
||||||
- Pluggable author resolvers with priority chain and confidence levels
|
- Pluggable author resolvers with priority chain and confidence levels
|
||||||
- Per-file attribution recorded in the commit message
|
- Per-file attribution recorded in the commit message
|
||||||
@@ -24,6 +24,7 @@ See `functional specification.md` for the full specification.
|
|||||||
- Repository-level lock with stale-lock detection (stored outside the repo)
|
- Repository-level lock with stale-lock detection (stored outside the repo)
|
||||||
- Rotating file logs plus console logging
|
- Rotating file logs plus console logging
|
||||||
- Dry-run mode, exit codes suitable for external monitoring
|
- Dry-run mode, exit codes suitable for external monitoring
|
||||||
|
- `export` command: the collected history as analysis-ready CSV/JSONL (one row per file change, with author, timestamps and Tracker-Run identifiers)
|
||||||
- Cross-platform: Windows / Linux / macOS
|
- Cross-platform: Windows / Linux / macOS
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
@@ -42,6 +43,7 @@ git-auto-tracker run [--dry-run] # collect changes and commit
|
|||||||
git-auto-tracker status # show detected changes without committing
|
git-auto-tracker status # show detected changes without committing
|
||||||
git-auto-tracker validate # validate configuration / repository / resolvers
|
git-auto-tracker validate # validate configuration / repository / resolvers
|
||||||
git-auto-tracker config # print effective configuration
|
git-auto-tracker config # print effective configuration
|
||||||
|
git-auto-tracker export -o out.csv # export the commit history for analysis
|
||||||
```
|
```
|
||||||
|
|
||||||
Global options:
|
Global options:
|
||||||
@@ -52,6 +54,25 @@ Global options:
|
|||||||
--version Print version and exit
|
--version Print version and exit
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Analyzing the audit trail
|
||||||
|
|
||||||
|
`export` turns the tracker commits of the configured repository into a
|
||||||
|
flat table — one row per file change, with the collection time, run
|
||||||
|
identifier, host, change letter, path, author and last-modified time:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git-auto-tracker export -o history.csv # CSV (default format)
|
||||||
|
git-auto-tracker export -o history.jsonl # JSON Lines
|
||||||
|
git-auto-tracker export -o history.csv --since 2026-09-01
|
||||||
|
```
|
||||||
|
|
||||||
|
The rows load directly into pandas/Excel for analysis (who changes most,
|
||||||
|
which subtrees churn, change volume per day, everything ever deleted).
|
||||||
|
Commits written with a different `commit.file_separator` than the current
|
||||||
|
configuration are skipped with a warning, since their fields cannot be
|
||||||
|
split reliably. Non-tracker commits (e.g. the repository's initial
|
||||||
|
import) are ignored automatically.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
See [`config/example.yaml`](config/example.yaml). Configuration is YAML;
|
See [`config/example.yaml`](config/example.yaml). Configuration is YAML;
|
||||||
|
|||||||
@@ -278,6 +278,8 @@ Glob-style matching should be supported.
|
|||||||
|
|
||||||
Files excluded by configuration shall neither be committed nor included in commit messages.
|
Files excluded by configuration shall neither be committed nor included in commit messages.
|
||||||
|
|
||||||
|
The application log shall record every excluded file together with the rule that matched it (e.g. `matched exclude 'runtime/**'`) at DEBUG level. The INFO-level summary shall contain only the number of in-scope and excluded changes, so normal-run logs stay compact while investigations can switch on DEBUG (Section 22).
|
||||||
|
|
||||||
Files ignored by the repository's own `.gitignore` shall also be excluded. The tracker shall not override ignore rules unless explicitly configured to do so.
|
Files ignored by the repository's own `.gitignore` shall also be excluded. The tracker shall not override ignore rules unless explicitly configured to do so.
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -713,6 +715,7 @@ repository
|
|||||||
number of detected changes
|
number of detected changes
|
||||||
files processed
|
files processed
|
||||||
author resolution result
|
author resolution result
|
||||||
|
excluded files and the rule that matched them (DEBUG level, Section 7)
|
||||||
Git operations
|
Git operations
|
||||||
warnings
|
warnings
|
||||||
errors
|
errors
|
||||||
|
|||||||
@@ -204,8 +204,17 @@ class ApplicationController:
|
|||||||
change_set = self._records_from_status(entries)
|
change_set = self._records_from_status(entries)
|
||||||
|
|
||||||
kept, excluded = filter_change_set(change_set, self.config.filters)
|
kept, excluded = filter_change_set(change_set, self.config.filters)
|
||||||
if excluded:
|
# Two-tier reporting (Section 7): the INFO summary shows whether
|
||||||
log.info("Excluded by filters: %d file(s)", len(excluded))
|
# filters bit; per-file paths and matching rules go to DEBUG so
|
||||||
|
# investigations get the details without bloating normal runs.
|
||||||
|
log.info(
|
||||||
|
"Detected %d change(s): %d in scope, %d excluded by filters",
|
||||||
|
len(change_set),
|
||||||
|
len(kept),
|
||||||
|
len(excluded),
|
||||||
|
)
|
||||||
|
for exclusion in excluded:
|
||||||
|
log.debug("Excluded: %s (%s)", exclusion.path, exclusion.reason)
|
||||||
|
|
||||||
# Self-exclusion safety net (FR-028): operational artefacts are
|
# Self-exclusion safety net (FR-028): operational artefacts are
|
||||||
# already outside the repository; belt-and-braces path check here.
|
# already outside the repository; belt-and-braces path check here.
|
||||||
@@ -223,7 +232,7 @@ class ApplicationController:
|
|||||||
started_at=dt.datetime.now().astimezone(),
|
started_at=dt.datetime.now().astimezone(),
|
||||||
host=socket.gethostname(),
|
host=socket.gethostname(),
|
||||||
)
|
)
|
||||||
log.info("Detected %d changes (run %s)", len(kept), run.run_id)
|
log.info("Collection run %s", run.run_id)
|
||||||
|
|
||||||
for record in [*kept.added, *kept.modified]:
|
for record in [*kept.added, *kept.modified]:
|
||||||
result = self.resolvers.resolve(self.config.repository_path, record.path, log=log)
|
result = self.resolvers.resolve(self.config.repository_path, record.path, log=log)
|
||||||
@@ -231,6 +240,16 @@ class ApplicationController:
|
|||||||
log.info(
|
log.info(
|
||||||
"Author resolved: %s -> %s", record.path, result.identifier
|
"Author resolved: %s -> %s", record.path, result.identifier
|
||||||
)
|
)
|
||||||
|
# Resolution details (source, confidence) go to DEBUG: they are
|
||||||
|
# operational diagnostics, not part of the audit record.
|
||||||
|
log.debug(
|
||||||
|
"Author resolution detail: %s -> source=%s confidence=%s",
|
||||||
|
record.path,
|
||||||
|
result.source,
|
||||||
|
result.confidence.value
|
||||||
|
if hasattr(result.confidence, "value")
|
||||||
|
else result.confidence,
|
||||||
|
)
|
||||||
|
|
||||||
message = build_commit_message(
|
message = build_commit_message(
|
||||||
kept,
|
kept,
|
||||||
|
|||||||
@@ -1,13 +1,16 @@
|
|||||||
"""File filter and security validation (Sections 7 and 33).
|
"""File filter and security validation (Sections 7 and 33).
|
||||||
|
|
||||||
Pure-path matching: patterns are matched against the repository-relative
|
Pure-path matching: patterns are matched against the repository-relative
|
||||||
POSIX path. Excluded files are neither staged nor committed nor reported.
|
POSIX path. Excluded files are neither staged nor committed nor reported;
|
||||||
Forbidden files abort the commit with a security exit code by default.
|
their paths and the rule that matched them are logged at DEBUG level
|
||||||
|
(Section 7, Section 22). Forbidden files abort the commit with a security
|
||||||
|
exit code by default.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from git_auto_tracker.changes.models import ChangeSet
|
from git_auto_tracker.changes.models import ChangeSet
|
||||||
from git_auto_tracker.config import FileFilters, SecurityConfig
|
from git_auto_tracker.config import FileFilters, SecurityConfig
|
||||||
@@ -16,6 +19,14 @@ from git_auto_tracker.errors import SecurityError
|
|||||||
_PATTERN_CACHE: dict[str, re.Pattern[str]] = {}
|
_PATTERN_CACHE: dict[str, re.Pattern[str]] = {}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Exclusion:
|
||||||
|
"""One file kept out of the ChangeSet, with the reason (Section 7)."""
|
||||||
|
|
||||||
|
path: str
|
||||||
|
reason: str # e.g. "matched exclude 'runtime/**'" or "no include pattern matched"
|
||||||
|
|
||||||
|
|
||||||
def matches(path: str, pattern: str) -> bool:
|
def matches(path: str, pattern: str) -> bool:
|
||||||
"""Match a repository-relative POSIX path against a glob pattern.
|
"""Match a repository-relative POSIX path against a glob pattern.
|
||||||
|
|
||||||
@@ -93,26 +104,41 @@ def _translate_segment(segment: str) -> str:
|
|||||||
|
|
||||||
def is_included(path: str, filters: FileFilters) -> bool:
|
def is_included(path: str, filters: FileFilters) -> bool:
|
||||||
"""True when the path matches any include pattern (empty include = all)."""
|
"""True when the path matches any include pattern (empty include = all)."""
|
||||||
return not filters.include or any(matches(path, p) for p in filters.include)
|
return inclusion_reason(path, filters) is None
|
||||||
|
|
||||||
|
|
||||||
|
def inclusion_reason(path: str, filters: FileFilters) -> str | None:
|
||||||
|
"""None when included; otherwise the reason this path is out of scope."""
|
||||||
|
if not filters.include or any(matches(path, p) for p in filters.include):
|
||||||
|
return None
|
||||||
|
return "no include pattern matched"
|
||||||
|
|
||||||
|
|
||||||
def is_excluded(path: str, filters: FileFilters) -> bool:
|
def is_excluded(path: str, filters: FileFilters) -> bool:
|
||||||
return any(matches(path, p) for p in filters.exclude)
|
return exclusion_reason(path, filters) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def exclusion_reason(path: str, filters: FileFilters) -> str | None:
|
||||||
|
"""None when not excluded; otherwise the exclude pattern that matched."""
|
||||||
|
for pattern in filters.exclude:
|
||||||
|
if matches(path, pattern):
|
||||||
|
return f"matched exclude '{pattern}'"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def filter_change_set(
|
def filter_change_set(
|
||||||
change_set: ChangeSet,
|
change_set: ChangeSet,
|
||||||
filters: FileFilters,
|
filters: FileFilters,
|
||||||
) -> tuple[ChangeSet, list[str]]:
|
) -> tuple[ChangeSet, list[Exclusion]]:
|
||||||
"""Apply include/exclude rules to a ChangeSet.
|
"""Apply include/exclude rules to a ChangeSet.
|
||||||
|
|
||||||
Returns the filtered ChangeSet plus the list of excluded paths
|
Returns the filtered ChangeSet plus one Exclusion per file kept out,
|
||||||
(for logging).
|
carrying the path and the rule that matched (for DEBUG logging).
|
||||||
"""
|
"""
|
||||||
from git_auto_tracker.changes.models import ChangeType
|
from git_auto_tracker.changes.models import ChangeType
|
||||||
|
|
||||||
kept = ChangeSet()
|
kept = ChangeSet()
|
||||||
excluded: list[str] = []
|
excluded: list[Exclusion] = []
|
||||||
|
|
||||||
buckets = {
|
buckets = {
|
||||||
ChangeType.ADDED: (change_set.added, kept.added),
|
ChangeType.ADDED: (change_set.added, kept.added),
|
||||||
@@ -124,10 +150,18 @@ def filter_change_set(
|
|||||||
for records, target in buckets.values():
|
for records, target in buckets.values():
|
||||||
for record in records:
|
for record in records:
|
||||||
path = record.previous_path or record.path
|
path = record.previous_path or record.path
|
||||||
if is_included(path, filters) and not is_excluded(path, filters):
|
# Exclude matches are checked first: when a path is both
|
||||||
|
# out of include scope and exclude-matched, the explicit
|
||||||
|
# exclude rule is the more actionable reason to report.
|
||||||
|
blocked = exclusion_reason(path, filters)
|
||||||
|
if blocked is not None:
|
||||||
|
excluded.append(Exclusion(record.path, blocked))
|
||||||
|
continue
|
||||||
|
out_of_scope = inclusion_reason(path, filters)
|
||||||
|
if out_of_scope is not None:
|
||||||
|
excluded.append(Exclusion(record.path, out_of_scope))
|
||||||
|
continue
|
||||||
target.append(record)
|
target.append(record)
|
||||||
else:
|
|
||||||
excluded.append(record.path)
|
|
||||||
|
|
||||||
return kept, excluded
|
return kept, excluded
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,17 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
subparsers.add_parser("status", help="show detected changes without committing")
|
subparsers.add_parser("status", help="show detected changes without committing")
|
||||||
subparsers.add_parser("validate", help="validate configuration, repository and resolvers")
|
subparsers.add_parser("validate", help="validate configuration, repository and resolvers")
|
||||||
subparsers.add_parser("config", help="print the effective configuration (secrets redacted)")
|
subparsers.add_parser("config", help="print the effective configuration (secrets redacted)")
|
||||||
|
|
||||||
|
export_parser = subparsers.add_parser(
|
||||||
|
"export", help="export the tracker commit history for analysis"
|
||||||
|
)
|
||||||
|
export_parser.add_argument(
|
||||||
|
"-o", "--output", type=Path, required=True,
|
||||||
|
help="output file (.csv or .jsonl); format follows the suffix",
|
||||||
|
)
|
||||||
|
export_parser.add_argument(
|
||||||
|
"--since", help="only commits more recent than this date (e.g. 2026-09-01)",
|
||||||
|
)
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
@@ -74,6 +85,20 @@ def _config_command(config_path: Path) -> int:
|
|||||||
return ExitCode.SUCCESS
|
return ExitCode.SUCCESS
|
||||||
|
|
||||||
|
|
||||||
|
def _export_command(controller: ApplicationController, args: argparse.Namespace) -> int:
|
||||||
|
"""export command body: emit the audit trail as CSV/JSONL."""
|
||||||
|
from git_auto_tracker.exporting.history import export_history
|
||||||
|
|
||||||
|
count = export_history(
|
||||||
|
controller.config,
|
||||||
|
args.output,
|
||||||
|
since=args.since,
|
||||||
|
log=logging.getLogger("git_auto_tracker"),
|
||||||
|
)
|
||||||
|
print(f"{count} file-change record(s) exported to {args.output}")
|
||||||
|
return ExitCode.SUCCESS
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
def main(argv: list[str] | None = None) -> int:
|
||||||
args = build_parser().parse_args(argv)
|
args = build_parser().parse_args(argv)
|
||||||
|
|
||||||
@@ -100,6 +125,9 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
if args.command == "validate":
|
if args.command == "validate":
|
||||||
return _validate_command(controller)
|
return _validate_command(controller)
|
||||||
|
|
||||||
|
if args.command == "export":
|
||||||
|
return _export_command(controller, args)
|
||||||
|
|
||||||
if args.command == "status":
|
if args.command == "status":
|
||||||
lock.acquire()
|
lock.acquire()
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
"""History export for analysis (README: analyzing the audit trail)."""
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
"""History export for analysis (README: analyzing the audit trail).
|
||||||
|
|
||||||
|
Fetches tracker commits with `git log -z` (NUL-separated records, UTF-8;
|
||||||
|
Section 6.2 rules applied in reverse), parses the structured commit
|
||||||
|
messages built by message_builder (Sections 15/16/36), and emits one row
|
||||||
|
per file change ("long format") as CSV or JSON Lines.
|
||||||
|
|
||||||
|
The one-line entries use the SAME separator as collection did
|
||||||
|
(commit.file_separator). Commits created with a different separator than
|
||||||
|
the currently configured one are skipped with a warning, because their
|
||||||
|
fields cannot be split reliably.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from git_auto_tracker.config import AppConfig
|
||||||
|
from git_auto_tracker.errors import GitError
|
||||||
|
|
||||||
|
#: Field delimiter for the git log --format string; cannot occur in
|
||||||
|
#: commit messages produced by the tracker.
|
||||||
|
_UNIT_SEPARATOR = "\x1f"
|
||||||
|
|
||||||
|
_SUBJECT_PREFIX = "SCADA automatic change capture - "
|
||||||
|
_FIELDS = ("commit", "collected_at", "run_id", "host", "letter",
|
||||||
|
"path", "previous_path", "author", "mtime")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class HistoryRecord:
|
||||||
|
"""One commit, parsed (collection-level fields + per-file rows)."""
|
||||||
|
|
||||||
|
commit: str
|
||||||
|
collected_at: str = ""
|
||||||
|
run_id: str = ""
|
||||||
|
host: str = ""
|
||||||
|
rows: list[dict[str, str]] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_commits(repo: Path, since: str | None = None) -> list[tuple[str, str]]:
|
||||||
|
"""Return (commit hash, full message) tuples, newest first.
|
||||||
|
|
||||||
|
Uses -z (NUL-separated records): multi-line bodies cannot corrupt the
|
||||||
|
record boundaries. Raises GitError when git fails.
|
||||||
|
"""
|
||||||
|
cmd = ["git", "log", "-z", f"--format=%H{_UNIT_SEPARATOR}%B"]
|
||||||
|
if since:
|
||||||
|
cmd.append(f"--since={since}")
|
||||||
|
try:
|
||||||
|
raw = subprocess.run(
|
||||||
|
cmd, cwd=str(repo), capture_output=True, check=True
|
||||||
|
).stdout
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
stderr = (exc.stderr or b"").decode("utf-8", errors="replace").strip()
|
||||||
|
raise GitError(f"git log failed: {stderr or exc.returncode}") from exc
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise GitError("git executable not found on PATH") from exc
|
||||||
|
|
||||||
|
pairs: list[tuple[str, str]] = []
|
||||||
|
for entry in raw.split(b"\x00"):
|
||||||
|
if not entry:
|
||||||
|
continue
|
||||||
|
commit_hash, _, body = entry.decode("utf-8").partition(_UNIT_SEPARATOR)
|
||||||
|
pairs.append((commit_hash, body))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def parse_message(message: str, separator: str) -> HistoryRecord | None:
|
||||||
|
"""Parse one tracker commit message; None when it is not one of ours.
|
||||||
|
|
||||||
|
Non-tracker commits (e.g. the repository's initial import) return None.
|
||||||
|
Messages written with a DIFFERENT separator than *separator* are
|
||||||
|
recognized as tracker commits but yield no rows, since their fields
|
||||||
|
cannot be split reliably; the caller reports them as skipped.
|
||||||
|
"""
|
||||||
|
record = HistoryRecord(commit="")
|
||||||
|
saw_tracker_format = False
|
||||||
|
lines = message.splitlines()
|
||||||
|
for line in lines:
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped.startswith(_SUBJECT_PREFIX):
|
||||||
|
record.collected_at = stripped[len(_SUBJECT_PREFIX):].strip()
|
||||||
|
saw_tracker_format = True
|
||||||
|
elif stripped.startswith("Tracker-Run: "):
|
||||||
|
record.run_id = stripped.split(": ", 1)[1].strip()
|
||||||
|
saw_tracker_format = True
|
||||||
|
elif stripped.startswith("Tracker-Host: "):
|
||||||
|
record.host = stripped.split(": ", 1)[1].strip()
|
||||||
|
saw_tracker_format = True
|
||||||
|
elif line.startswith(" ") and f" {separator} " in line:
|
||||||
|
saw_tracker_format = True
|
||||||
|
record.rows.append(_parse_change_line(line.strip(), separator))
|
||||||
|
if not saw_tracker_format:
|
||||||
|
return None
|
||||||
|
return record
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_change_line(text: str, separator: str) -> dict[str, str]:
|
||||||
|
"""Parse one ` <letter> <sep> <path> <sep> ...` entry (Section 15)."""
|
||||||
|
parts = [p.strip() for p in text.split(f" {separator} ")]
|
||||||
|
row = {
|
||||||
|
"letter": parts[0],
|
||||||
|
"path": parts[1] if len(parts) > 1 else "",
|
||||||
|
"previous_path": "",
|
||||||
|
"author": "",
|
||||||
|
"mtime": "",
|
||||||
|
}
|
||||||
|
letter = row["letter"]
|
||||||
|
if letter in ("A", "M") and len(parts) == 4:
|
||||||
|
row["author"], row["mtime"] = parts[2], parts[3]
|
||||||
|
elif letter == "R" and "->" in row["path"]:
|
||||||
|
old, new = row["path"].split(" -> ", 1)
|
||||||
|
row["previous_path"], row["path"] = old, new
|
||||||
|
# D lines stay bare (2 parts); malformed lines keep empty fields
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def to_rows(records: list[HistoryRecord]) -> list[dict[str, str]]:
|
||||||
|
"""Flatten parsed commits to long format: one row per file change."""
|
||||||
|
rows: list[dict[str, str]] = []
|
||||||
|
for record in records:
|
||||||
|
for row in record.rows:
|
||||||
|
rows.append({
|
||||||
|
"commit": record.commit,
|
||||||
|
"collected_at": record.collected_at,
|
||||||
|
"run_id": record.run_id,
|
||||||
|
"host": record.host,
|
||||||
|
**row,
|
||||||
|
})
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def write_csv(rows: list[dict[str, str]], output: Path) -> None:
|
||||||
|
with output.open("w", encoding="utf-8", newline="") as handle:
|
||||||
|
writer = csv.DictWriter(handle, fieldnames=list(_FIELDS))
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def write_jsonl(rows: list[dict[str, str]], output: Path) -> None:
|
||||||
|
with output.open("w", encoding="utf-8", newline="\n") as handle:
|
||||||
|
for row in rows:
|
||||||
|
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def export_history(
|
||||||
|
config: AppConfig,
|
||||||
|
output: Path,
|
||||||
|
since: str | None = None,
|
||||||
|
log: object | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Export all tracker commits of the configured repository to a file.
|
||||||
|
|
||||||
|
Returns the number of exported file-change rows. The output format is
|
||||||
|
chosen from the file suffix (.jsonl -> JSON Lines, otherwise CSV).
|
||||||
|
"""
|
||||||
|
separator = config.commit.file_separator
|
||||||
|
repo = config.repository_path
|
||||||
|
|
||||||
|
exported: list[HistoryRecord] = []
|
||||||
|
skipped_other_separator: list[str] = []
|
||||||
|
for commit_hash, message in fetch_commits(repo, since):
|
||||||
|
record = parse_message(message, separator)
|
||||||
|
if record is None:
|
||||||
|
continue # not a tracker commit (e.g. initial import)
|
||||||
|
record.commit = commit_hash
|
||||||
|
if not record.rows and record.collected_at:
|
||||||
|
# Tracker commit whose entries used a different separator
|
||||||
|
skipped_other_separator.append(commit_hash[:8])
|
||||||
|
continue
|
||||||
|
exported.append(record)
|
||||||
|
|
||||||
|
if skipped_other_separator and log is not None:
|
||||||
|
log.warning(
|
||||||
|
"export: %d commit(s) used a different file separator and were "
|
||||||
|
"skipped: %s",
|
||||||
|
len(skipped_other_separator),
|
||||||
|
", ".join(skipped_other_separator),
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = to_rows(exported)
|
||||||
|
if output.suffix == ".jsonl":
|
||||||
|
write_jsonl(rows, output)
|
||||||
|
else:
|
||||||
|
write_csv(rows, output)
|
||||||
|
return len(rows)
|
||||||
@@ -295,6 +295,88 @@ class TestExtraction:
|
|||||||
assert before == after
|
assert before == after
|
||||||
|
|
||||||
|
|
||||||
|
class TestExportHistory:
|
||||||
|
"""export subcommand: collected history becomes analysis-ready rows."""
|
||||||
|
|
||||||
|
def _collect_two_cycles(self, git_repo: Path, config: dict, tmp_path: Path) -> None:
|
||||||
|
# Cycle 1: one addition with sidecar attribution
|
||||||
|
(git_repo / "config.xml").write_text("<c/>", encoding="utf-8")
|
||||||
|
(git_repo / "config.xml.metadata").write_text(
|
||||||
|
"last_modified_by=DOMAIN\\jsmith\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
_run(config, tmp_path)
|
||||||
|
# Cycle 2: one deletion (bare D line in the history)
|
||||||
|
(git_repo / "config.xml").unlink()
|
||||||
|
_run(config, tmp_path)
|
||||||
|
|
||||||
|
def test_export_csv_round_trip(
|
||||||
|
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
import csv
|
||||||
|
|
||||||
|
from git_auto_tracker.config import AppConfig
|
||||||
|
from git_auto_tracker.exporting.history import export_history
|
||||||
|
|
||||||
|
self._collect_two_cycles(git_repo, tracker_config, tmp_path)
|
||||||
|
app_config = AppConfig.from_dict(tracker_config, config_dir=tmp_path)
|
||||||
|
output = tmp_path / "history.csv"
|
||||||
|
count = export_history(app_config, output)
|
||||||
|
|
||||||
|
# 3 rows: A config.xml + A config.xml.metadata, then D config.xml
|
||||||
|
assert count == 3
|
||||||
|
with output.open(encoding="utf-8", newline="") as handle:
|
||||||
|
rows = list(csv.DictReader(handle))
|
||||||
|
assert rows[0]["letter"] == "D" # newest collection first
|
||||||
|
assert rows[0]["path"] == "config.xml"
|
||||||
|
assert rows[1]["letter"] == "A"
|
||||||
|
assert rows[1]["path"] == "config.xml"
|
||||||
|
assert rows[1]["author"] == "DOMAIN\\jsmith"
|
||||||
|
assert rows[1]["run_id"] # trailer parsed
|
||||||
|
|
||||||
|
def test_export_jsonl_and_since_filter(
|
||||||
|
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
import json
|
||||||
|
|
||||||
|
from git_auto_tracker.config import AppConfig
|
||||||
|
from git_auto_tracker.exporting.history import export_history
|
||||||
|
|
||||||
|
self._collect_two_cycles(git_repo, tracker_config, tmp_path)
|
||||||
|
app_config = AppConfig.from_dict(tracker_config, config_dir=tmp_path)
|
||||||
|
output = tmp_path / "history.jsonl"
|
||||||
|
|
||||||
|
# Future-dated --since excludes everything (2099, not 2999: Git's
|
||||||
|
# approxidate parser mishandles far-future years and returns all)
|
||||||
|
assert export_history(app_config, output, since="2099-01-01") == 0
|
||||||
|
assert output.read_text(encoding="utf-8") == ""
|
||||||
|
|
||||||
|
# No filter: both collections' rows present, newest first
|
||||||
|
assert export_history(app_config, output) == 3
|
||||||
|
lines = output.read_text(encoding="utf-8").splitlines()
|
||||||
|
parsed = [json.loads(line) for line in lines]
|
||||||
|
assert parsed[0]["letter"] == "D"
|
||||||
|
assert parsed[1]["letter"] == "A"
|
||||||
|
assert parsed[0]["collected_at"] >= parsed[-1]["collected_at"]
|
||||||
|
|
||||||
|
def test_cli_export_command(
|
||||||
|
self, git_repo: Path, tracker_config: dict, tmp_path: Path, capsys
|
||||||
|
) -> None:
|
||||||
|
from git_auto_tracker.cli import main
|
||||||
|
|
||||||
|
self._collect_two_cycles(git_repo, tracker_config, tmp_path)
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
config_path = tmp_path / "cfg.yaml"
|
||||||
|
config_path.write_text(yaml.safe_dump(tracker_config), encoding="utf-8")
|
||||||
|
output = tmp_path / "out.csv"
|
||||||
|
|
||||||
|
exit_code = main(["--config", str(config_path), "export", "-o", str(output)])
|
||||||
|
assert exit_code == 0
|
||||||
|
assert output.is_file()
|
||||||
|
captured = capsys.readouterr().out
|
||||||
|
assert "3 file-change record(s)" in captured
|
||||||
|
|
||||||
|
|
||||||
class TestSafetyAborts:
|
class TestSafetyAborts:
|
||||||
def test_pre_existing_staged_changes_abort(
|
def test_pre_existing_staged_changes_abort(
|
||||||
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
self, git_repo: Path, tracker_config: dict, tmp_path: Path
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""Tests for exclusion reporting and log-level policy (Sections 7 and 22).
|
||||||
|
|
||||||
|
Excluded paths and the rule that matched them are logged at DEBUG level;
|
||||||
|
the INFO summary carries only the in-scope/excluded split. Commit messages
|
||||||
|
never contain exclusion details (Section 7).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from git_auto_tracker.changes.filters import (
|
||||||
|
Exclusion,
|
||||||
|
exclusion_reason,
|
||||||
|
filter_change_set,
|
||||||
|
inclusion_reason,
|
||||||
|
)
|
||||||
|
from git_auto_tracker.changes.models import ChangeRecord, ChangeSet, ChangeType
|
||||||
|
from git_auto_tracker.config import FileFilters
|
||||||
|
|
||||||
|
|
||||||
|
class TestReasonFunctions:
|
||||||
|
def test_exclusion_reason_names_the_pattern(self) -> None:
|
||||||
|
filters = FileFilters(include=["**/*"], exclude=["runtime/**", "**/*.log"])
|
||||||
|
assert exclusion_reason("runtime/cache.dat", filters) == "matched exclude 'runtime/**'"
|
||||||
|
assert exclusion_reason("logs/a.log", filters) == "matched exclude '**/*.log'"
|
||||||
|
assert exclusion_reason("keep.xml", filters) is None
|
||||||
|
|
||||||
|
def test_inclusion_reason_when_out_of_scope(self) -> None:
|
||||||
|
filters = FileFilters(include=["Stations/**"], exclude=[])
|
||||||
|
assert inclusion_reason("System/global.xml", filters) == "no include pattern matched"
|
||||||
|
assert inclusion_reason("Stations/PS1.xml", filters) is None
|
||||||
|
|
||||||
|
def test_empty_include_means_everything_included(self) -> None:
|
||||||
|
filters = FileFilters(include=[], exclude=["**/*.tmp"])
|
||||||
|
assert inclusion_reason("anything.xml", filters) is None
|
||||||
|
|
||||||
|
def test_legacy_predicates_still_work(self) -> None:
|
||||||
|
filters = FileFilters(include=["**/*"], exclude=["**/*.log"])
|
||||||
|
from git_auto_tracker.changes.filters import is_excluded, is_included
|
||||||
|
|
||||||
|
assert is_included("a.xml", filters) is True
|
||||||
|
assert is_excluded("a.log", filters) is True
|
||||||
|
assert is_excluded("a.xml", filters) is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestFilterChangeSet:
|
||||||
|
def _change_set(self) -> ChangeSet:
|
||||||
|
return ChangeSet(
|
||||||
|
added=[
|
||||||
|
ChangeRecord(path="runtime/cache.dat", change_type=ChangeType.ADDED),
|
||||||
|
ChangeRecord(path="Stations/PS1.xml", change_type=ChangeType.ADDED),
|
||||||
|
],
|
||||||
|
modified=[
|
||||||
|
ChangeRecord(path="logs/app.log", change_type=ChangeType.MODIFIED),
|
||||||
|
ChangeRecord(path="System/global.xml", change_type=ChangeType.MODIFIED),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_exclusions_carry_reasons(self) -> None:
|
||||||
|
filters = FileFilters(include=["**/*"], exclude=["runtime/**", "**/*.log"])
|
||||||
|
kept, excluded = filter_change_set(self._change_set(), filters)
|
||||||
|
|
||||||
|
assert [r.path for r in kept.added] == ["Stations/PS1.xml"]
|
||||||
|
assert [r.path for r in kept.modified] == ["System/global.xml"]
|
||||||
|
by_path = {e.path: e.reason for e in excluded}
|
||||||
|
assert by_path["runtime/cache.dat"] == "matched exclude 'runtime/**'"
|
||||||
|
assert by_path["logs/app.log"] == "matched exclude '**/*.log'"
|
||||||
|
|
||||||
|
def test_out_of_scope_excluded_with_include_reason(self) -> None:
|
||||||
|
filters = FileFilters(include=["Stations/**"], exclude=["**/*.log"])
|
||||||
|
_, excluded = filter_change_set(self._change_set(), filters)
|
||||||
|
by_path = {e.path: e.reason for e in excluded}
|
||||||
|
assert by_path["System/global.xml"] == "no include pattern matched"
|
||||||
|
assert by_path["runtime/cache.dat"] == "no include pattern matched"
|
||||||
|
assert by_path["logs/app.log"] == "matched exclude '**/*.log'"
|
||||||
|
|
||||||
|
def test_exclusion_is_dataclass(self) -> None:
|
||||||
|
record = Exclusion(path="x.tmp", reason="matched exclude '**/*.tmp'")
|
||||||
|
assert record.path == "x.tmp"
|
||||||
|
assert "x.tmp" in record.reason or "**/*.tmp" in record.reason
|
||||||
|
|
||||||
|
|
||||||
|
class TestLogLevels:
|
||||||
|
"""INFO summary only; per-file detail at DEBUG (application wiring)."""
|
||||||
|
|
||||||
|
def test_collect_logs_split_and_debug_details(
|
||||||
|
self, git_repo, tracker_config, tmp_path, caplog
|
||||||
|
) -> None:
|
||||||
|
from git_auto_tracker.application import ApplicationController
|
||||||
|
from git_auto_tracker.config import AppConfig
|
||||||
|
from git_auto_tracker.logging.setup import setup_logging
|
||||||
|
|
||||||
|
(git_repo / "junk.log").write_text("noise", encoding="utf-8")
|
||||||
|
(git_repo / "real.xml").write_text("<r/>", encoding="utf-8")
|
||||||
|
|
||||||
|
config = AppConfig.from_dict(tracker_config, config_dir=tmp_path)
|
||||||
|
setup_logging(
|
||||||
|
config.logging,
|
||||||
|
config_dir=tmp_path,
|
||||||
|
repository_path=config.repository_path,
|
||||||
|
override_level="DEBUG",
|
||||||
|
)
|
||||||
|
# The app logger does not propagate (setup_logging), so caplog's
|
||||||
|
# root-attached handler never sees the records; attach it directly.
|
||||||
|
app_logger = logging.getLogger("git_auto_tracker")
|
||||||
|
app_logger.addHandler(caplog.handler)
|
||||||
|
try:
|
||||||
|
controller = ApplicationController(config)
|
||||||
|
outcome = controller.collect()
|
||||||
|
finally:
|
||||||
|
app_logger.removeHandler(caplog.handler)
|
||||||
|
|
||||||
|
assert outcome.change_count == 1 # real.xml only
|
||||||
|
debug_text = caplog.text
|
||||||
|
|
||||||
|
# INFO summary: the split, no per-file exclusion detail
|
||||||
|
assert "Detected 2 change(s): 1 in scope, 1 excluded by filters" in debug_text
|
||||||
|
# DEBUG detail: path plus the matching rule
|
||||||
|
assert "Excluded: junk.log (matched exclude '**/*.log')" in debug_text
|
||||||
|
# Author resolution details land at DEBUG too
|
||||||
|
assert "Author resolution detail: real.xml -> source=" in debug_text
|
||||||
|
# The exclusion must NOT appear in the commit message (Section 7)
|
||||||
|
assert "junk.log" not in (outcome.commit_message or "")
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
"""Unit tests for history export parsing (exporting.history).
|
||||||
|
|
||||||
|
The commit-message grammar is defined by message_builder (Section 15);
|
||||||
|
these tests pin the parser against it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from git_auto_tracker.exporting.history import (
|
||||||
|
HistoryRecord,
|
||||||
|
parse_message,
|
||||||
|
to_rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
_MESSAGE = """SCADA automatic change capture - 2026-09-12 08:00:03+02:00
|
||||||
|
|
||||||
|
Summary:
|
||||||
|
Added: 1
|
||||||
|
Modified: 1
|
||||||
|
Deleted: 1
|
||||||
|
Renamed: 0
|
||||||
|
|
||||||
|
Changes:
|
||||||
|
A ; Stations/PS101/pump2.xml ; DOMAIN\\jsmith ; 2026-09-12 07:41:22+02:00
|
||||||
|
M ; System/global.xml ; UNKNOWN ; 2026-09-12 07:45:01+02:00
|
||||||
|
D ; Stations/PS027/old-config.xml
|
||||||
|
|
||||||
|
Tracker-Version: 1.0.0
|
||||||
|
Tracker-Run: 01M2ADPMSHSA2TZKVH6PSVDVA0
|
||||||
|
Tracker-Host: SCADA-SRV-01
|
||||||
|
Tracker-Changes: 3
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseMessage:
|
||||||
|
def test_full_message(self) -> None:
|
||||||
|
record = parse_message(_MESSAGE, ";")
|
||||||
|
assert record is not None
|
||||||
|
assert record.collected_at == "2026-09-12 08:00:03+02:00"
|
||||||
|
assert record.run_id == "01M2ADPMSHSA2TZKVH6PSVDVA0"
|
||||||
|
assert record.host == "SCADA-SRV-01"
|
||||||
|
assert len(record.rows) == 3
|
||||||
|
|
||||||
|
added = record.rows[0]
|
||||||
|
assert added["letter"] == "A"
|
||||||
|
assert added["path"] == "Stations/PS101/pump2.xml"
|
||||||
|
assert added["author"] == "DOMAIN\\jsmith"
|
||||||
|
assert added["mtime"] == "2026-09-12 07:41:22+02:00"
|
||||||
|
|
||||||
|
modified = record.rows[1]
|
||||||
|
assert modified["letter"] == "M"
|
||||||
|
assert modified["author"] == "UNKNOWN"
|
||||||
|
|
||||||
|
deleted = record.rows[2]
|
||||||
|
assert deleted["letter"] == "D"
|
||||||
|
assert deleted["path"] == "Stations/PS027/old-config.xml"
|
||||||
|
assert deleted["author"] == "" # bare line (FR-013)
|
||||||
|
|
||||||
|
def test_paths_with_semicolon_inside_are_not_broken(self) -> None:
|
||||||
|
# A path containing ';' is ambiguous by design; the parser splits on
|
||||||
|
# ' ; ' so a lone ';' inside a filename keeps the line intact here.
|
||||||
|
message = _MESSAGE.replace(
|
||||||
|
"Stations/PS101/pump2.xml", " Stations/PS101;backup.xml".strip()
|
||||||
|
)
|
||||||
|
record = parse_message(message, ";")
|
||||||
|
assert record is not None
|
||||||
|
assert record.rows[0]["path"] == "Stations/PS101;backup.xml"
|
||||||
|
|
||||||
|
def test_non_tracker_message_returns_none(self) -> None:
|
||||||
|
assert parse_message("initial import\n", ";") is None
|
||||||
|
assert parse_message("", ";") is None
|
||||||
|
|
||||||
|
def test_custom_separator(self) -> None:
|
||||||
|
message = _MESSAGE.replace(" ; ", " | ")
|
||||||
|
record = parse_message(message, "|")
|
||||||
|
assert record is not None
|
||||||
|
assert len(record.rows) == 3
|
||||||
|
assert record.rows[0]["path"] == "Stations/PS101/pump2.xml"
|
||||||
|
|
||||||
|
def test_wrong_separator_yields_no_rows_but_is_tracker_commit(self) -> None:
|
||||||
|
# Written with ';', parsed with '|': recognized but not splittable.
|
||||||
|
record = parse_message(_MESSAGE, "|")
|
||||||
|
assert record is not None
|
||||||
|
assert record.collected_at # tracker commit detected
|
||||||
|
assert record.rows == [] # but no reliable rows
|
||||||
|
|
||||||
|
def test_rename_line(self) -> None:
|
||||||
|
message = _MESSAGE.replace(
|
||||||
|
" D ; Stations/PS027/old-config.xml",
|
||||||
|
" R ; Stations/PS027/old.xml -> Stations/PS027/new.xml",
|
||||||
|
)
|
||||||
|
record = parse_message(message, ";")
|
||||||
|
assert record is not None
|
||||||
|
row = record.rows[2]
|
||||||
|
assert row["letter"] == "R"
|
||||||
|
assert row["previous_path"] == "Stations/PS027/old.xml"
|
||||||
|
assert row["path"] == "Stations/PS027/new.xml"
|
||||||
|
|
||||||
|
|
||||||
|
class TestToRows:
|
||||||
|
def test_long_format(self) -> None:
|
||||||
|
record = parse_message(_MESSAGE, ";")
|
||||||
|
assert record is not None
|
||||||
|
record.commit = "a1b2c3d"
|
||||||
|
rows = to_rows([record])
|
||||||
|
assert len(rows) == 3
|
||||||
|
first = rows[0]
|
||||||
|
assert first["commit"] == "a1b2c3d"
|
||||||
|
assert first["run_id"] == "01M2ADPMSHSA2TZKVH6PSVDVA0"
|
||||||
|
assert first["letter"] == "A"
|
||||||
|
assert set(first) == {
|
||||||
|
"commit", "collected_at", "run_id", "host",
|
||||||
|
"letter", "path", "previous_path", "author", "mtime",
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_empty(self) -> None:
|
||||||
|
assert to_rows([]) == []
|
||||||
|
assert to_rows([HistoryRecord(commit="x")]) == []
|
||||||
Reference in New Issue
Block a user