66 lines
2.2 KiB
Python
66 lines
2.2 KiB
Python
"""Laedt die Konfiguration (tools.yaml, filter_rules.json)."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
from .model import Source, Tool
|
|
|
|
|
|
def load_tools(path: str | Path) -> tuple[list[Tool], list[Source]]:
|
|
"""Liest tools.yaml (Tools) und general.yaml (allgemeines Wissen).
|
|
|
|
general-Quellen werden bevorzugt aus config/general.yaml geladen; als Fallback
|
|
aus einem 'general:'-Abschnitt in tools.yaml (Rueckwaertskompatibilitaet).
|
|
Rueckgabe: (tools, general_sources)
|
|
"""
|
|
path = Path(path)
|
|
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
tools = [Tool.from_dict(t) for t in data.get("tools", [])]
|
|
|
|
general_file = path.parent / "general.yaml"
|
|
if general_file.exists():
|
|
gdata = yaml.safe_load(general_file.read_text(encoding="utf-8")) or {}
|
|
general = [Source.from_dict(s) for s in gdata.get("general", [])]
|
|
else:
|
|
general = [Source.from_dict(s) for s in data.get("general", [])]
|
|
return tools, general
|
|
|
|
|
|
def load_chunking_defaults(path: str | Path) -> dict:
|
|
"""Liest config/chunking.yaml -> defaults-dict (gemeinsame Chunk-Optionen).
|
|
|
|
Fehlt die Datei, wird ein leeres dict geliefert; die endgueltigen Optionen
|
|
entstehen ohnehin in chunker.effective_opts (DEFAULTS < diese < per-Quelle).
|
|
"""
|
|
p = Path(path)
|
|
if not p.exists():
|
|
return {}
|
|
data = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
|
|
return dict(data.get("defaults", {}) or {})
|
|
|
|
|
|
def load_filter_rules(path: str | Path) -> dict:
|
|
p = Path(path)
|
|
if not p.exists():
|
|
return {"blacklist_keywords": [], "drop_css_classes": [], "regex_redact": []}
|
|
return json.loads(p.read_text(encoding="utf-8"))
|
|
|
|
|
|
def load_approvals(path: str | Path) -> dict:
|
|
"""Liest config/approvals.yaml -> {'urls': set, 'hashes': set}."""
|
|
p = Path(path)
|
|
urls: set[str] = set()
|
|
hashes: set[str] = set()
|
|
if p.exists():
|
|
data = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
|
|
for entry in data.get("approved", []) or []:
|
|
e = str(entry).strip()
|
|
if e.startswith("hash:"):
|
|
hashes.add(e[5:].strip())
|
|
elif e:
|
|
urls.add(e)
|
|
return {"urls": urls, "hashes": hashes}
|