510 lines
19 KiB
Python
510 lines
19 KiB
Python
"""Liest den vorhandenen Wissensbestand vom Datentraeger (statt aus dem Run-Report).
|
||
|
||
Wichtig fuer inkrementelle Laeufe: der review_report.json enthaelt nur die im
|
||
letzten Lauf verarbeiteten Dokumente. Die GitLab-Pages-Seite soll aber den GESAMTEN
|
||
Bestand sehen -> hier aus output/processed (approved) und staging/pending
|
||
(pending) laden.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
|
||
def parse_frontmatter(text: str) -> tuple[dict, str]:
|
||
"""Einfacher YAML-Frontmatter-Parser (key: value, plus [..]-Listen)."""
|
||
if not text.startswith("---"):
|
||
return {}, text
|
||
end = text.find("\n---", 3)
|
||
if end == -1:
|
||
return {}, text
|
||
fm_raw = text[3:end].strip("\n")
|
||
body = text[end + 4:].lstrip("\n")
|
||
meta: dict = {}
|
||
for line in fm_raw.split("\n"):
|
||
if ":" not in line:
|
||
continue
|
||
key, _, val = line.partition(":")
|
||
key = key.strip()
|
||
val = val.strip()
|
||
if val.startswith("[") and val.endswith("]"):
|
||
items = [v.strip().strip('"') for v in val[1:-1].split(",") if v.strip()]
|
||
meta[key] = items
|
||
else:
|
||
meta[key] = val.strip('"')
|
||
return meta, body
|
||
|
||
|
||
def _fmt_list(items) -> str:
|
||
return "[" + ", ".join(f'"{str(i)}"' for i in items) + "]"
|
||
|
||
|
||
def patch_frontmatter_file(path, *, tags=None, owners=None, contact=None,
|
||
meta_fingerprint=None) -> bool:
|
||
"""Aktualisiert einzelne Frontmatter-Felder einer bestehenden .md IN-PLACE.
|
||
|
||
Aendert nur die angegebenen Zeilen (Rest des Dokuments bleibt unangetastet).
|
||
Genutzt fuer inkrementelles Re-Tagging: aendern sich Metadaten in der Config,
|
||
werden bestehende Dateien guenstig angepasst statt neu geladen.
|
||
|
||
Felder, die den Ablagepfad bestimmen (scope/domain), werden bewusst NICHT
|
||
angefasst. Gibt True zurueck, wenn die Datei geschrieben wurde.
|
||
"""
|
||
from pathlib import Path as _Path
|
||
|
||
p = _Path(path)
|
||
text = p.read_text(encoding="utf-8")
|
||
if not text.startswith("---"):
|
||
return False
|
||
end = text.find("\n---", 3)
|
||
if end == -1:
|
||
return False
|
||
head = text[:end]
|
||
rest = text[end:] # beginnt mit "\n---" (schliessender Block + Body)
|
||
lines = head.split("\n")
|
||
|
||
updates: dict[str, str] = {}
|
||
if tags is not None:
|
||
updates["tags"] = f"tags: {_fmt_list(tags)}"
|
||
if owners is not None:
|
||
updates["owners"] = f"owners: {_fmt_list(owners)}"
|
||
if contact is not None:
|
||
updates["contact"] = f'contact: "{contact}"'
|
||
if meta_fingerprint is not None:
|
||
updates["meta_fingerprint"] = f'meta_fingerprint: "{meta_fingerprint}"'
|
||
|
||
for key, rendered in updates.items():
|
||
prefix = f"{key}:"
|
||
for i, ln in enumerate(lines):
|
||
if ln.startswith(prefix):
|
||
lines[i] = rendered
|
||
break
|
||
else:
|
||
lines.append(rendered)
|
||
|
||
p.write_text("\n".join(lines) + rest, encoding="utf-8")
|
||
return True
|
||
|
||
|
||
def reconcile_meta(paths, source, current_fp: str) -> int:
|
||
"""Bringt die Metadaten bestehender Dateien auf den Stand der Quelle.
|
||
|
||
Pro Datei werden Tags (domain:/tool:/scope: + Quell-Tags), owners, contact und
|
||
der meta_fingerprint neu geschrieben. Die Tags werden mit der jeweils im File
|
||
hinterlegten domain/tool/scope rekonstruiert (identisch zu apply_tags).
|
||
"""
|
||
from .transformers.tagger import compute_tags
|
||
|
||
n = 0
|
||
for path in paths:
|
||
try:
|
||
meta, _ = parse_frontmatter(Path(path).read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
tags = compute_tags(
|
||
meta.get("domain", source.domain),
|
||
meta.get("tool", "allgemein"),
|
||
meta.get("scope", source.scope.value),
|
||
list(source.tags),
|
||
)
|
||
if patch_frontmatter_file(
|
||
path,
|
||
tags=tags,
|
||
owners=list(source.owners),
|
||
contact=source.contact_address,
|
||
meta_fingerprint=current_fp,
|
||
):
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def prune_scope_orphans(data_dir, produced: dict) -> list[str]:
|
||
"""Loescht verwaiste processed-Dateien nach einem Scope-Wechsel.
|
||
|
||
produced: {(domain, url): {scopes...}} = die in DIESEM Lauf erzeugten
|
||
approved-Scopes je (Domaene, URL). Eine bestehende Datei wird nur geloescht,
|
||
wenn ihr (domain, url) gerade neu erzeugt wurde, ihr Scope aber NICHT mehr
|
||
dabei ist (also die Ablage gewechselt hat). Domaenen-intern -> kollisionssicher;
|
||
laeuft erst NACH erfolgreichem Schreiben -> kein Datenverlust bei Abruffehlern.
|
||
"""
|
||
base = Path(data_dir) / "processed"
|
||
deleted: list[str] = []
|
||
if not base.exists():
|
||
return deleted
|
||
domains = {dom for (dom, _) in produced}
|
||
seen: set = set()
|
||
for dom in domains:
|
||
for md in base.glob(f"*/{dom}/**/*.md"):
|
||
if md in seen:
|
||
continue
|
||
seen.add(md)
|
||
try:
|
||
meta, _ = parse_frontmatter(md.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
scopes = produced.get((meta.get("domain", ""), meta.get("url", "")))
|
||
if not scopes:
|
||
continue
|
||
if meta.get("scope", "") not in scopes:
|
||
try:
|
||
md.unlink()
|
||
except OSError:
|
||
continue
|
||
deleted.append(str(md))
|
||
return deleted
|
||
|
||
|
||
def processed_signature(data_dir) -> tuple[str, int, dict]:
|
||
"""Inhalts-Signatur des freigegebenen Bestands (CI-sicher, ohne Datei-mtimes).
|
||
|
||
Hasht je Dokument (relativer Pfad + content_hash + meta_fingerprint), damit
|
||
JEDE Aenderung erkannt wird: neue/entfernte Datei, geaenderter Inhalt ODER
|
||
geaenderte Metadaten (Re-Tagging). Liefert (signature, anzahl, je_scope).
|
||
"""
|
||
base = Path(data_dir) / "processed"
|
||
items: list[str] = []
|
||
by_scope: dict[str, int] = {}
|
||
count = 0
|
||
if base.exists():
|
||
for md in sorted(base.rglob("*.md")):
|
||
try:
|
||
meta, _ = parse_frontmatter(md.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
rel = str(md.relative_to(base))
|
||
items.append(f"{rel}|{meta.get('content_hash', '')}|{meta.get('meta_fingerprint', '')}")
|
||
sc = meta.get("scope", "")
|
||
by_scope[sc] = by_scope.get(sc, 0) + 1
|
||
count += 1
|
||
sig = hashlib.sha256("\n".join(items).encode("utf-8")).hexdigest()[:16]
|
||
return sig, count, by_scope
|
||
|
||
|
||
def write_run_meta(data_dir, summary: dict | None = None) -> Path:
|
||
"""Schreibt `output/_meta.json` fuer nachgelagerte Systeme.
|
||
|
||
Liegt bewusst auf `output/`-Ebene (nicht im `processed/`-Feed): globale Status-Info
|
||
ueber den GESAMTEN Bestand, kein Wissensdokument.
|
||
|
||
Felder: `last_run` (dieser Lauf), `last_change` (wann sich der Bestand zuletzt
|
||
inhaltlich/metadatenseitig geaendert hat – bleibt stehen, wenn der Lauf nichts
|
||
geaendert hat), `documents`, `by_scope`, `content_signature` und optional die
|
||
Status-Zusammenfassung des letzten Laufs.
|
||
"""
|
||
root = Path(data_dir)
|
||
meta_path = root / "_meta.json"
|
||
prev: dict = {}
|
||
if meta_path.exists():
|
||
try:
|
||
prev = json.loads(meta_path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
prev = {}
|
||
|
||
sig, docs, by_scope = processed_signature(data_dir)
|
||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||
if prev.get("content_signature") == sig and prev.get("last_change"):
|
||
last_change = prev["last_change"]
|
||
else:
|
||
last_change = now
|
||
|
||
payload = {
|
||
"schema": 1,
|
||
"last_run": now,
|
||
"last_change": last_change,
|
||
"documents": docs,
|
||
"by_scope": by_scope,
|
||
"content_signature": sig,
|
||
}
|
||
if summary is not None:
|
||
payload["last_run_summary"] = summary
|
||
|
||
root.mkdir(parents=True, exist_ok=True)
|
||
meta_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||
return meta_path
|
||
|
||
|
||
def build_index(data_dir) -> dict:
|
||
"""Katalog ueber den freigegebenen Bestand: pro Voll-Dokument (output/processed)
|
||
Herkunft, Aenderungsstand und - falls vorhanden - die zugehoerigen Chunks und
|
||
Anhang-Dokumente (eingebundene PDFs). Auch gechunkte Anhang-PDFs
|
||
(`chunk_kind: attachment`) erhalten ihren `chunks`-Block und zaehlen in `chunks_total`.
|
||
|
||
Beantwortet fuer nachgelagerte Systeme (und Menschen): was liegt pro Domaene/Tool
|
||
wo, wann zuletzt geaendert, und gibt es das Dokument zusaetzlich als Chunks/Anhaenge?
|
||
Pfade sind relativ zum Output-Root (`processed/...`, `chunks/...`).
|
||
"""
|
||
from .model import page_identity
|
||
|
||
base = Path(data_dir)
|
||
processed = base / "processed"
|
||
chunks_root = base / "chunks"
|
||
|
||
# 1) Alle Dokumente einmal einlesen (Chunks ausgenommen)
|
||
items: list[tuple] = [] # (md, meta)
|
||
if processed.exists():
|
||
for md in sorted(processed.rglob("*.md")):
|
||
try:
|
||
meta, _ = parse_frontmatter(md.read_text(encoding="utf-8"))
|
||
except OSError:
|
||
continue
|
||
if meta.get("kind") == "chunk":
|
||
continue
|
||
items.append((md, meta))
|
||
|
||
# 2) Anhang-Dokumente nach Elternseite gruppieren ((scope, domaene, parent-identity))
|
||
attachments_by_parent: dict[tuple, list[dict]] = {}
|
||
for md, meta in items:
|
||
if meta.get("kind") != "attachment":
|
||
continue
|
||
key = (meta.get("scope", ""), meta.get("domain", ""), page_identity(meta.get("parent_url", "")))
|
||
attachments_by_parent.setdefault(key, []).append({
|
||
"name": meta.get("attachment_name", "") or md.stem,
|
||
"path": str(md.relative_to(base)),
|
||
})
|
||
|
||
documents: list[dict] = []
|
||
by_domain: dict[str, dict] = {}
|
||
by_scope: dict[str, int] = {}
|
||
total_chunks = 0
|
||
total_attachments = 0
|
||
|
||
for md, meta in items:
|
||
scope = meta.get("scope", "")
|
||
domain = meta.get("domain", "")
|
||
tool = meta.get("tool", "")
|
||
kind = meta.get("kind", "document") or "document"
|
||
|
||
entry = {
|
||
"scope": scope,
|
||
"domain": domain,
|
||
"tool": tool,
|
||
"kind": kind,
|
||
"title": meta.get("part") or meta.get("title") or md.stem,
|
||
"url": meta.get("url", ""),
|
||
"path": str(md.relative_to(base)),
|
||
"content_hash": meta.get("content_hash", ""),
|
||
"last_updated": meta.get("last_updated", ""),
|
||
}
|
||
|
||
# Chunks dieser Datei (gleicher relativer Pfad in output/chunks/<...>/<stem>/).
|
||
# Gilt fuer Voll-Dokumente UND gechunkte Anhang-PDFs (chunk_kind: attachment) -
|
||
# daher vor der kind-Fallunterscheidung, damit auch Anhaenge ihren chunks-Block
|
||
# erhalten und in chunks_total mitzaehlen.
|
||
rel_parent = md.relative_to(processed).parent
|
||
chunk_dir = chunks_root / rel_parent / md.stem
|
||
chunk_files = sorted(chunk_dir.glob("*.md")) if chunk_dir.is_dir() else []
|
||
chunks_info = None
|
||
if chunk_files:
|
||
try:
|
||
cmeta, _ = parse_frontmatter(chunk_files[0].read_text(encoding="utf-8"))
|
||
except OSError:
|
||
cmeta = {}
|
||
chunks_info = {
|
||
"count": len(chunk_files),
|
||
"path": str(chunk_dir.relative_to(base)),
|
||
"fingerprint": cmeta.get("chunk_fingerprint", ""),
|
||
}
|
||
total_chunks += len(chunk_files)
|
||
entry["chunks"] = chunks_info
|
||
|
||
if kind == "attachment":
|
||
entry["parent_url"] = meta.get("parent_url", "")
|
||
total_attachments += 1
|
||
else:
|
||
# zugehoerige Anhang-Dokumente (eingebundene PDFs)
|
||
akey = (scope, domain, page_identity(meta.get("url", "")))
|
||
entry["attachments"] = attachments_by_parent.get(akey) or None
|
||
|
||
documents.append(entry)
|
||
|
||
d = by_domain.setdefault(domain, {"documents": 0, "chunks": 0, "attachments": 0,
|
||
"scopes": set(), "tools": set()})
|
||
d["documents"] += 1
|
||
if kind == "attachment":
|
||
d["attachments"] += 1
|
||
if entry.get("chunks"):
|
||
d["chunks"] += entry["chunks"]["count"]
|
||
d["scopes"].add(scope)
|
||
d["tools"].add(tool)
|
||
by_scope[scope] = by_scope.get(scope, 0) + 1
|
||
|
||
by_domain_out = {
|
||
dom: {
|
||
"documents": v["documents"],
|
||
"chunks": v["chunks"],
|
||
"attachments": v["attachments"],
|
||
"scopes": sorted(v["scopes"]),
|
||
"tools": sorted(v["tools"]),
|
||
}
|
||
for dom, v in sorted(by_domain.items())
|
||
}
|
||
|
||
return {
|
||
"schema": 1,
|
||
"documents_total": len(documents),
|
||
"chunks_total": total_chunks,
|
||
"attachments_total": total_attachments,
|
||
"by_scope": dict(sorted(by_scope.items())),
|
||
"by_domain": by_domain_out,
|
||
"documents": documents,
|
||
}
|
||
|
||
|
||
def write_index(data_dir) -> Path:
|
||
"""Schreibt `output/_index.json` (dokument-genauer Katalog inkl. Chunks).
|
||
|
||
Liegt bewusst auf `output/`-Ebene (nicht im `processed/`-Feed): Katalog ueber den
|
||
gesamten Bestand, kein Wissensdokument.
|
||
"""
|
||
root = Path(data_dir)
|
||
root.mkdir(parents=True, exist_ok=True)
|
||
path = root / "_index.json"
|
||
payload = build_index(data_dir)
|
||
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def append_run_log(data_dir, entry: dict, cap: int = 500) -> Path:
|
||
"""Haengt einen Lauf-Eintrag an `output/run_log.jsonl` an (append-only Historie).
|
||
|
||
Bewusst auf `output/`-Ebene, eine JSON-Zeile pro ETL-Lauf. Im Gegensatz zu
|
||
`_meta.json`/`_index.json` ist dies KEIN deterministischer Snapshot, sondern ein
|
||
Verlauf (jeder Lauf = ein Ereignis mit Zeitstempel) -> so sieht man historisch, ob
|
||
Laeufe sauber durchliefen und welche Quellen Fehler hatten. Auf die letzten `cap`
|
||
Eintraege gekappt, damit die Datei nicht unbegrenzt waechst.
|
||
"""
|
||
root = Path(data_dir)
|
||
root.mkdir(parents=True, exist_ok=True)
|
||
path = root / "run_log.jsonl"
|
||
record = {"ts": datetime.now(timezone.utc).isoformat(timespec="seconds"), **entry}
|
||
|
||
lines = path.read_text(encoding="utf-8").splitlines() if path.exists() else []
|
||
lines.append(json.dumps(record, ensure_ascii=False))
|
||
if cap and len(lines) > cap:
|
||
lines = lines[-cap:]
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def read_last_run(data_dir) -> dict:
|
||
"""Liefert den letzten Eintrag aus `output/run_log.jsonl` (oder {} wenn keiner)."""
|
||
path = Path(data_dir) / "run_log.jsonl"
|
||
if not path.exists():
|
||
return {}
|
||
last = ""
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
if line.strip():
|
||
last = line
|
||
try:
|
||
return json.loads(last) if last else {}
|
||
except json.JSONDecodeError:
|
||
return {}
|
||
|
||
|
||
def prune_duplicate_files(data_dir) -> list[str]:
|
||
"""Entfernt verwaiste Hash-Suffix-Duplikate im Feed.
|
||
|
||
Frueher konnten mehrere Quellen DIESELBE Seite liefern -> `<slug>.md` plus
|
||
`<slug>-<hash>.md`. Mit dem Dedup im Review-Gate entsteht das nicht mehr neu; diese
|
||
Funktion raeumt Altbestaende auf: Ein `<slug>-<hash>.md` wird geloescht, wenn die
|
||
Basisdatei `<slug>.md` im selben Ordner DIESELBE Seite referenziert (page_identity).
|
||
Verschiedene Seiten mit zufaellig gleichem Titel (unterschiedliche page_identity)
|
||
bleiben erhalten.
|
||
"""
|
||
import re as _re
|
||
|
||
from .model import page_identity
|
||
|
||
base = Path(data_dir) / "processed"
|
||
removed: list[str] = []
|
||
if not base.exists():
|
||
return removed
|
||
suffix_re = _re.compile(r"^(?P<slug>.+)-[0-9a-f]{16}\.md$")
|
||
for md in sorted(base.rglob("*.md")):
|
||
m = suffix_re.match(md.name)
|
||
if not m:
|
||
continue
|
||
sibling = md.with_name(m.group("slug") + ".md")
|
||
if not sibling.exists():
|
||
continue
|
||
try:
|
||
meta_a, _ = parse_frontmatter(md.read_text(encoding="utf-8"))
|
||
meta_b, _ = parse_frontmatter(sibling.read_text(encoding="utf-8"))
|
||
except OSError:
|
||
continue
|
||
id_a = page_identity(meta_a.get("url", ""))
|
||
id_b = page_identity(meta_b.get("url", ""))
|
||
if id_a and id_a == id_b:
|
||
try:
|
||
md.unlink()
|
||
except OSError:
|
||
continue
|
||
removed.append(str(md.relative_to(Path(data_dir))))
|
||
return removed
|
||
|
||
|
||
def load_documents(data_dir: str, staging_dir: str | None = None) -> list[dict]:
|
||
"""Liefert alle Dokumente aus `output/processed/` und `staging/pending/` (intern)
|
||
als Liste von dicts.
|
||
|
||
Jedes dict: domain, tool, scope, tags, title, part, url, status, path, text.
|
||
Der Status kommt aus dem Frontmatter (review_status) bzw. dem staging-Ordner.
|
||
|
||
`data_dir`: Output-Verzeichnis (Feed). `staging_dir`: top-level Staging-Verzeichnis.
|
||
Wenn nicht angegeben, wird `<data_dir>/staging` als Fallback geprueft (Test-Convenience).
|
||
"""
|
||
base = Path(data_dir)
|
||
staging_root = Path(staging_dir) if staging_dir else (base / "staging")
|
||
docs: list[dict] = []
|
||
|
||
for root, default_status in ((base / "processed", "approved"), (staging_root, None)):
|
||
if not root.exists():
|
||
continue
|
||
for md in sorted(root.rglob("*.md")):
|
||
try:
|
||
meta, body = parse_frontmatter(md.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
status = meta.get("review_status") or default_status or "pending"
|
||
from .quality import quality_breakdown
|
||
qb = quality_breakdown(body)
|
||
docs.append(
|
||
{
|
||
"domain": meta.get("domain", ""),
|
||
"tool": meta.get("tool", ""),
|
||
"scope": meta.get("scope", "intern"),
|
||
"kind": meta.get("kind", "document") or "document",
|
||
"tags": meta.get("tags", []),
|
||
"owners": meta.get("owners", []),
|
||
"contact": meta.get("contact", ""),
|
||
"title": meta.get("part") or meta.get("title") or md.stem,
|
||
"part": meta.get("part", ""),
|
||
"url": meta.get("url", ""),
|
||
"hash": meta.get("content_hash", ""),
|
||
"notes": meta.get("review_notes", ""),
|
||
"quality": qb["total"],
|
||
"quality_parts": qb,
|
||
"status": status,
|
||
"path": _relpath_from_roots(md, base, staging_root),
|
||
"text": body.strip(),
|
||
}
|
||
)
|
||
return docs
|
||
|
||
|
||
def _relpath_from_roots(md, output_root, staging_root) -> str:
|
||
"""Liefert einen verstaendlichen relativen Pfad: 'output/...' oder 'staging/...'.
|
||
staging zuerst pruefen, damit ein verschachteltes Layout (staging unter output,
|
||
z.B. in Tests) korrekt als 'staging/...' erkannt wird.
|
||
Faellt zurueck auf den absoluten String, falls die Datei in keinem der Roots liegt."""
|
||
try:
|
||
return "staging/" + str(md.relative_to(staging_root))
|
||
except ValueError:
|
||
pass
|
||
try:
|
||
return "output/" + str(md.relative_to(output_root))
|
||
except ValueError:
|
||
return str(md)
|