Merge commit 'cfaf67010017eab368216aded483a64126dbcb2e' as 'bahn/wissensdatenbank'

bahn/wissensdatenbank/src/config_loader.py

@@ -0,0 +1,65 @@
+"""Laedt die Konfiguration (tools.yaml, filter_rules.json)."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import yaml
+
+from .model import Source, Tool
+
+
+def load_tools(path: str | Path) -> tuple[list[Tool], list[Source]]:
+    """Liest tools.yaml (Tools) und general.yaml (allgemeines Wissen).
+
+    general-Quellen werden bevorzugt aus config/general.yaml geladen; als Fallback
+    aus einem 'general:'-Abschnitt in tools.yaml (Rueckwaertskompatibilitaet).
+    Rueckgabe: (tools, general_sources)
+    """
+    path = Path(path)
+    data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    tools = [Tool.from_dict(t) for t in data.get("tools", [])]
+
+    general_file = path.parent / "general.yaml"
+    if general_file.exists():
+        gdata = yaml.safe_load(general_file.read_text(encoding="utf-8")) or {}
+        general = [Source.from_dict(s) for s in gdata.get("general", [])]
+    else:
+        general = [Source.from_dict(s) for s in data.get("general", [])]
+    return tools, general
+
+
+def load_chunking_defaults(path: str | Path) -> dict:
+    """Liest config/chunking.yaml -> defaults-dict (gemeinsame Chunk-Optionen).
+
+    Fehlt die Datei, wird ein leeres dict geliefert; die endgueltigen Optionen
+    entstehen ohnehin in chunker.effective_opts (DEFAULTS < diese < per-Quelle).
+    """
+    p = Path(path)
+    if not p.exists():
+        return {}
+    data = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
+    return dict(data.get("defaults", {}) or {})
+
+
+def load_filter_rules(path: str | Path) -> dict:
+    p = Path(path)
+    if not p.exists():
+        return {"blacklist_keywords": [], "drop_css_classes": [], "regex_redact": []}
+    return json.loads(p.read_text(encoding="utf-8"))
+
+
+def load_approvals(path: str | Path) -> dict:
+    """Liest config/approvals.yaml -> {'urls': set, 'hashes': set}."""
+    p = Path(path)
+    urls: set[str] = set()
+    hashes: set[str] = set()
+    if p.exists():
+        data = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
+        for entry in data.get("approved", []) or []:
+            e = str(entry).strip()
+            if e.startswith("hash:"):
+                hashes.add(e[5:].strip())
+            elif e:
+                urls.add(e)
+    return {"urls": urls, "hashes": hashes}