feat: implement knowledge management system (spec complete, all 53 tasks done)
This commit is contained in:
@@ -680,6 +680,10 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
# --- uninstall-hooks ---
|
||||
subparsers.add_parser("uninstall-hooks", help="Git-Hooks entfernen")
|
||||
|
||||
# --- knowledge (Subcommand-Gruppe) ---
|
||||
from monorepo.knowledge.cli import register_knowledge_subcommand
|
||||
register_knowledge_subcommand(subparsers)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
@@ -703,6 +707,7 @@ _COMMAND_HANDLERS = {
|
||||
"fed-status": _cmd_fed_status,
|
||||
"install-hooks": _cmd_install_hooks,
|
||||
"uninstall-hooks": _cmd_uninstall_hooks,
|
||||
"knowledge": "_knowledge", # Sentinel – handled specially in main()
|
||||
}
|
||||
|
||||
|
||||
@@ -722,6 +727,13 @@ def main() -> None:
|
||||
parser.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
# Special handling for knowledge subcommand (has sub-subparsers)
|
||||
if args.command == "knowledge":
|
||||
from monorepo.knowledge.cli import handle_knowledge_command
|
||||
root = _find_monorepo_root()
|
||||
exit_code = handle_knowledge_command(args, root)
|
||||
sys.exit(exit_code)
|
||||
|
||||
handler = _COMMAND_HANDLERS.get(args.command)
|
||||
if handler is None:
|
||||
parser.print_help()
|
||||
|
||||
@@ -8,6 +8,8 @@ Stellt den KnowledgeStore, YAMLIndex, IndexEntry und das Artefakt-Modell bereit:
|
||||
- Artifact/ArtifactMetadata: Vollständige Wissensartefakte mit YAML-Frontmatter
|
||||
- ETLPipeline: ETL-Pipeline für inkrementelle Verarbeitung
|
||||
- MarkdownSource: Quellstrategie für Markdown-Dateien
|
||||
- IngestionPipeline: Neuer Orchestrator für den vollständigen Ingestion-Workflow
|
||||
- QuickCapture: Schnelle Wissenserfassung über CLI
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
@@ -21,19 +23,29 @@ from monorepo.knowledge.artifact import (
|
||||
)
|
||||
from monorepo.knowledge.etl import ETLPipeline, IngestError, IngestResult
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline, IngestionResult
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource, SourceError
|
||||
from monorepo.knowledge.store import KnowledgeStore, SearchResult
|
||||
from monorepo.knowledge.store import (
|
||||
ContextInjectionResult,
|
||||
KnowledgeStore,
|
||||
SearchResult,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Artifact",
|
||||
"ArtifactLink",
|
||||
"ArtifactMetadata",
|
||||
"ContextInjectionResult",
|
||||
"ETLPipeline",
|
||||
"IndexEntry",
|
||||
"IngestError",
|
||||
"IngestResult",
|
||||
"IngestionPipeline",
|
||||
"IngestionResult",
|
||||
"KnowledgeStore",
|
||||
"MarkdownSource",
|
||||
"QuickCapture",
|
||||
"SearchResult",
|
||||
"SourceError",
|
||||
"YAMLIndex",
|
||||
|
||||
@@ -39,6 +39,10 @@ class ArtifactMetadata:
|
||||
|
||||
Enthält mindestens: Typ, Titel, Tags, Quellkontext, Erstelldatum,
|
||||
Aktualisierungsdatum, Teilbarkeits-Flag, Verknüpfungen, Content-Hash.
|
||||
|
||||
Knowledge-Management-Erweiterungen (alle optional mit Defaults für
|
||||
Rückwärtskompatibilität): source, category, people, projects,
|
||||
enrichment_pending, ocr_failed, fetch_failed.
|
||||
"""
|
||||
|
||||
type: str
|
||||
@@ -51,6 +55,17 @@ class ArtifactMetadata:
|
||||
links: list[ArtifactLink] = field(default_factory=list)
|
||||
content_hash: str = ""
|
||||
|
||||
# Knowledge Management extensions
|
||||
source: dict[str, str] = field(default_factory=dict)
|
||||
# e.g. {"type": "confluence", "url": "...", "space": "...", "file": "..."}
|
||||
|
||||
category: str = "" # meeting, decision, project, reference, link, inbox
|
||||
people: list[str] = field(default_factory=list) # [[people/name]] refs
|
||||
projects: list[str] = field(default_factory=list) # [[projects/slug]] refs
|
||||
enrichment_pending: bool = False
|
||||
ocr_failed: bool = False
|
||||
fetch_failed: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Artifact:
|
||||
@@ -186,6 +201,21 @@ def parse_frontmatter(file_path: Path) -> Artifact:
|
||||
raw: dict[str, Any] = yaml.safe_load(yaml_text) or {}
|
||||
|
||||
# Metadaten extrahieren
|
||||
raw_source = raw.get("source")
|
||||
source: dict[str, str] = {}
|
||||
if isinstance(raw_source, dict):
|
||||
source = {str(k): str(v) for k, v in raw_source.items()}
|
||||
|
||||
raw_people = raw.get("people")
|
||||
people: list[str] = (
|
||||
[str(p) for p in raw_people] if isinstance(raw_people, list) else []
|
||||
)
|
||||
|
||||
raw_projects = raw.get("projects")
|
||||
projects: list[str] = (
|
||||
[str(p) for p in raw_projects] if isinstance(raw_projects, list) else []
|
||||
)
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type=str(raw.get("type", "")),
|
||||
title=str(raw.get("title", "")),
|
||||
@@ -196,6 +226,13 @@ def parse_frontmatter(file_path: Path) -> Artifact:
|
||||
shareable=bool(raw.get("shareable", False)),
|
||||
links=_parse_links(raw.get("links")),
|
||||
content_hash=str(raw.get("content_hash", "")),
|
||||
source=source,
|
||||
category=str(raw.get("category", "")),
|
||||
people=people,
|
||||
projects=projects,
|
||||
enrichment_pending=bool(raw.get("enrichment_pending", False)),
|
||||
ocr_failed=bool(raw.get("ocr_failed", False)),
|
||||
fetch_failed=bool(raw.get("fetch_failed", False)),
|
||||
)
|
||||
|
||||
return Artifact(metadata=metadata, content=content, file_path=file_path)
|
||||
@@ -236,6 +273,28 @@ def _serialize_metadata(metadata: ArtifactMetadata) -> dict[str, Any]:
|
||||
if metadata.content_hash:
|
||||
data["content_hash"] = metadata.content_hash
|
||||
|
||||
# Knowledge Management extensions (nur wenn nicht-leer/nicht-default)
|
||||
if metadata.source:
|
||||
data["source"] = metadata.source
|
||||
|
||||
if metadata.category:
|
||||
data["category"] = metadata.category
|
||||
|
||||
if metadata.people:
|
||||
data["people"] = metadata.people
|
||||
|
||||
if metadata.projects:
|
||||
data["projects"] = metadata.projects
|
||||
|
||||
if metadata.enrichment_pending:
|
||||
data["enrichment_pending"] = metadata.enrichment_pending
|
||||
|
||||
if metadata.ocr_failed:
|
||||
data["ocr_failed"] = metadata.ocr_failed
|
||||
|
||||
if metadata.fetch_failed:
|
||||
data["fetch_failed"] = metadata.fetch_failed
|
||||
|
||||
return data
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,728 @@
|
||||
"""CLI-Schnittstelle für das Knowledge Management System.
|
||||
|
||||
Stellt das `knowledge`-Subcommand des Monorepo-CLI bereit mit folgenden
|
||||
Unterkommandos:
|
||||
- capture: Quick-Capture von Text, URLs oder Dateipfaden
|
||||
- ingest: Pipeline-Durchlauf starten
|
||||
- search: Suche über den Knowledge-Index
|
||||
- status: Übersicht über Inbox-Elemente und letzte Verarbeitung
|
||||
- link: Web-Link als Referenzmaterial speichern
|
||||
|
||||
Requirements: 15.1, 15.2, 15.3, 15.4, 15.5, 15.6, 15.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_CONTEXTS = ("bahn", "dhive", "privat")
|
||||
|
||||
_HELP_TEXT = """\
|
||||
Knowledge Management – Wissen erfassen, verarbeiten und durchsuchen.
|
||||
|
||||
Unterkommandos:
|
||||
capture Schnelle Wissenserfassung (Text, URL oder Dateipfad)
|
||||
ingest Ingestion-Pipeline für konfigurierte Quellen starten
|
||||
search Knowledge-Index durchsuchen
|
||||
status Übersicht über Inbox und Artefakte
|
||||
link Web-Link als Referenzmaterial speichern
|
||||
|
||||
Beispiele:
|
||||
monorepo knowledge capture "Meeting-Notiz: API-Design besprochen" --context bahn
|
||||
monorepo knowledge capture https://docs.example.com/api --tags api,referenz
|
||||
monorepo knowledge ingest --context dhive --dry-run
|
||||
monorepo knowledge search "API Design" --limit 5
|
||||
monorepo knowledge status --context privat
|
||||
monorepo knowledge link https://example.com --title "Beispiel" --tags ref
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _find_monorepo_root() -> Path:
|
||||
"""Find the monorepo root directory.
|
||||
|
||||
Delegates to the main CLI's root-finding logic.
|
||||
|
||||
Returns:
|
||||
Absolute path to the monorepo root.
|
||||
"""
|
||||
from monorepo.cli import _find_monorepo_root as _main_find_root
|
||||
|
||||
return _main_find_root()
|
||||
|
||||
|
||||
def _derive_context_from_cwd() -> str | None:
|
||||
"""Derive context from the current working directory.
|
||||
|
||||
Checks if the CWD path contains a known context segment
|
||||
(/bahn/, /dhive/, /privat/).
|
||||
|
||||
Returns:
|
||||
The context string or None if it cannot be determined.
|
||||
"""
|
||||
cwd = Path.cwd()
|
||||
path_str = str(cwd).replace("\\", "/")
|
||||
|
||||
for ctx in VALID_CONTEXTS:
|
||||
if f"/{ctx}/" in path_str or path_str.endswith(f"/{ctx}"):
|
||||
return ctx
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_context(context_flag: str | None) -> str:
|
||||
"""Resolve context from --context flag or current working directory.
|
||||
|
||||
Args:
|
||||
context_flag: Explicit context from CLI flag, or None.
|
||||
|
||||
Returns:
|
||||
The resolved context string.
|
||||
|
||||
Raises:
|
||||
SystemExit: If context cannot be determined.
|
||||
"""
|
||||
if context_flag:
|
||||
if context_flag not in VALID_CONTEXTS:
|
||||
print(
|
||||
f"Fehler: Ungültiger Kontext '{context_flag}'. "
|
||||
f"Erlaubt: {', '.join(VALID_CONTEXTS)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
return context_flag
|
||||
|
||||
derived = _derive_context_from_cwd()
|
||||
if derived:
|
||||
return derived
|
||||
|
||||
print(
|
||||
"Fehler: Kontext konnte nicht aus dem Arbeitsverzeichnis abgeleitet werden.\n"
|
||||
"Verwende --context <bahn|dhive|privat> um den Kontext explizit anzugeben.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parser Builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_knowledge_parser() -> argparse.ArgumentParser:
|
||||
"""Build a standalone argument parser for the knowledge subcommand.
|
||||
|
||||
This can be used independently for testing or as part of the
|
||||
main CLI parser registration.
|
||||
|
||||
Returns:
|
||||
ArgumentParser configured with all knowledge subcommands.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="knowledge",
|
||||
description=_HELP_TEXT,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(
|
||||
dest="knowledge_command",
|
||||
help="Knowledge-Unterkommandos",
|
||||
)
|
||||
|
||||
# --- knowledge capture ---
|
||||
p_capture = subparsers.add_parser(
|
||||
"capture",
|
||||
help="Text, URL oder Dateipfad schnell erfassen",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"input",
|
||||
help="Zu erfassender Text, URL oder Dateipfad.",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"--tags", "-t",
|
||||
default=None,
|
||||
help="Komma-getrennte Tags (z.B. 'api,meeting').",
|
||||
)
|
||||
|
||||
# --- knowledge ingest ---
|
||||
p_ingest = subparsers.add_parser(
|
||||
"ingest",
|
||||
help="Ingestion-Pipeline für konfigurierte Quellen starten",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Vorschau ohne Schreiboperationen.",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--verbose", "-v",
|
||||
action="store_true",
|
||||
help="Detaillierte Ausgabe.",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--no-commit",
|
||||
action="store_true",
|
||||
help="Git-Commit überspringen.",
|
||||
)
|
||||
|
||||
# --- knowledge search ---
|
||||
p_search = subparsers.add_parser(
|
||||
"search",
|
||||
help="Knowledge-Index durchsuchen",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"query",
|
||||
help="Suchbegriff.",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Suche auf einen Kontext beschränken.",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--limit", "-l",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Maximale Anzahl Ergebnisse (Standard: 20).",
|
||||
)
|
||||
|
||||
# --- knowledge status ---
|
||||
p_status = subparsers.add_parser(
|
||||
"status",
|
||||
help="Übersicht über Inbox-Elemente und Artefakte",
|
||||
)
|
||||
p_status.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Kontext anzeigen (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
|
||||
# --- knowledge link ---
|
||||
p_link = subparsers.add_parser(
|
||||
"link",
|
||||
help="Web-Link als Referenzmaterial speichern",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"url",
|
||||
help="Die zu speichernde URL.",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--title", "-T",
|
||||
default=None,
|
||||
help="Optionaler Titel für den Link.",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--tags", "-t",
|
||||
default=None,
|
||||
help="Komma-getrennte Tags (z.B. 'api,referenz').",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand Handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cmd_capture(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge capture <text|url|path> [--context CTX] [--tags TAG,TAG]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with input, context, tags.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for failure).
|
||||
"""
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
|
||||
root = _find_monorepo_root()
|
||||
context = _resolve_context(args.context)
|
||||
|
||||
# Parse tags
|
||||
tags: list[str] = []
|
||||
if args.tags:
|
||||
tags = [t.strip() for t in args.tags.split(",") if t.strip()]
|
||||
|
||||
capture = QuickCapture(monorepo_root=root)
|
||||
try:
|
||||
result_path = capture.capture(
|
||||
input_text=args.input,
|
||||
context=context,
|
||||
tags=tags,
|
||||
)
|
||||
print(f"✓ Erfasst: {result_path}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"Fehler beim Erfassen: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_ingest(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge ingest [--context CTX] [--dry-run] [--verbose] [--no-commit]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with context, dry_run, verbose, no_commit.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for failure).
|
||||
"""
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
|
||||
root = _find_monorepo_root()
|
||||
context = _resolve_context(args.context)
|
||||
|
||||
# Configure logging for verbose mode
|
||||
if args.verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(levelname)s: %(name)s: %(message)s",
|
||||
force=True,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=root,
|
||||
context=context,
|
||||
dry_run=args.dry_run,
|
||||
no_commit=args.no_commit,
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
print(f"[DRY RUN] Ingestion für Kontext '{context}'...")
|
||||
else:
|
||||
print(f"Ingestion für Kontext '{context}'...")
|
||||
|
||||
try:
|
||||
result = pipeline.run()
|
||||
except Exception as e:
|
||||
print(f"Fehler bei Ingestion: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Print results
|
||||
print(f"\n{'─' * 40}")
|
||||
print(f"Kontext: {result.context}")
|
||||
print(f"Verarbeitet: {result.processed}")
|
||||
print(f"Übersprungen: {result.skipped}")
|
||||
if hasattr(result, "tasks_created") and result.tasks_created:
|
||||
print(f"Tasks erstellt: {result.tasks_created}")
|
||||
if hasattr(result, "commit_sha") and result.commit_sha:
|
||||
print(f"Commit: {result.commit_sha[:8]}")
|
||||
|
||||
if result.errors:
|
||||
print(f"\n⚠ {len(result.errors)} Fehler:")
|
||||
for err in result.errors:
|
||||
print(f" • [{err.source}] {err.message}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_search(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge search <query> [--context CTX] [--limit N]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with query, context, limit.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for failure).
|
||||
"""
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
root = _find_monorepo_root()
|
||||
|
||||
# Determine allowed scopes based on context flag
|
||||
if args.context:
|
||||
if args.context not in VALID_CONTEXTS:
|
||||
print(
|
||||
f"Fehler: Ungültiger Kontext '{args.context}'.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
allowed_scopes = [args.context]
|
||||
else:
|
||||
allowed_scopes = list(VALID_CONTEXTS) + ["shared"]
|
||||
|
||||
# Create KnowledgeStore
|
||||
ks_path = root / "shared" / "knowledge-store"
|
||||
scope_config = ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
})
|
||||
store = KnowledgeStore(base_path=ks_path, scope_config=scope_config)
|
||||
|
||||
query = args.query
|
||||
results = store.search_cross_context(query=query, allowed_scopes=allowed_scopes)
|
||||
|
||||
# Apply limit
|
||||
limit = args.limit or 20
|
||||
results = results[:limit]
|
||||
|
||||
if not results:
|
||||
print(f"Keine Treffer für '{query}'.")
|
||||
return 0
|
||||
|
||||
print(f"Treffer für '{query}' ({len(results)} Ergebnis(se)):\n")
|
||||
for r in results:
|
||||
scope_label = r.entry.scope if r.entry.scope else "?"
|
||||
tags_str = ", ".join(r.entry.tags) if r.entry.tags else "-"
|
||||
entry_type = getattr(r.entry, "type", None) or "-"
|
||||
print(f" [{scope_label}] {r.entry.title}")
|
||||
print(f" Typ: {entry_type}")
|
||||
print(f" Tags: {tags_str}")
|
||||
print(f" Pfad: {r.entry.path}")
|
||||
print()
|
||||
|
||||
if results and results[0].partial_results:
|
||||
print("⚠ Suche wegen Timeout abgebrochen – möglicherweise unvollständig.")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_status(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge status [--context CTX]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with context.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success).
|
||||
"""
|
||||
root = _find_monorepo_root()
|
||||
context = _resolve_context(args.context)
|
||||
|
||||
knowledge_path = root / context / "knowledge"
|
||||
inbox_path = knowledge_path / "inbox"
|
||||
|
||||
# Count inbox items
|
||||
inbox_items: list[Path] = []
|
||||
if inbox_path.exists():
|
||||
inbox_items = [
|
||||
f for f in inbox_path.iterdir()
|
||||
if f.is_file() and f.suffix == ".md" and f.name != "index.md"
|
||||
]
|
||||
|
||||
# Count total artifacts across all subfolders
|
||||
total_artifacts = 0
|
||||
subfolders = ["meetings", "decisions", "projects", "people", "references", "links"]
|
||||
folder_counts: dict[str, int] = {}
|
||||
|
||||
for folder_name in subfolders:
|
||||
folder_path = knowledge_path / folder_name
|
||||
if folder_path.exists():
|
||||
count = sum(
|
||||
1 for f in folder_path.iterdir()
|
||||
if f.is_file() and f.suffix == ".md" and f.name != "index.md"
|
||||
)
|
||||
folder_counts[folder_name] = count
|
||||
total_artifacts += count
|
||||
|
||||
# Check _index.yaml
|
||||
index_path = knowledge_path / "_index.yaml"
|
||||
index_exists = index_path.exists()
|
||||
|
||||
# Check sources.yaml
|
||||
sources_path = knowledge_path / "sources.yaml"
|
||||
sources_exists = sources_path.exists()
|
||||
|
||||
# Output
|
||||
print(f"Knowledge Status: {context}")
|
||||
print(f"{'─' * 40}")
|
||||
print(f"Pfad: {knowledge_path}")
|
||||
print(f"Index: {'✓' if index_exists else '✗'}")
|
||||
print(f"Sources: {'✓' if sources_exists else '✗'}")
|
||||
print(f"\nInbox: {len(inbox_items)} Element(e)")
|
||||
print(f"Artefakte: {total_artifacts} gesamt")
|
||||
|
||||
if folder_counts:
|
||||
print()
|
||||
for folder, count in sorted(folder_counts.items()):
|
||||
if count > 0:
|
||||
print(f" {folder:12s} {count}")
|
||||
|
||||
if inbox_items:
|
||||
print(f"\nInbox-Elemente:")
|
||||
for item in sorted(inbox_items, key=lambda p: p.name, reverse=True)[:10]:
|
||||
print(f" • {item.name}")
|
||||
if len(inbox_items) > 10:
|
||||
print(f" ... und {len(inbox_items) - 10} weitere")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_link(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge link <url> [--title TITLE] [--tags TAG,TAG] [--context CTX]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with url, title, tags, context.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for failure).
|
||||
"""
|
||||
from monorepo.knowledge.sources.link import LinkRegistry
|
||||
|
||||
root = _find_monorepo_root()
|
||||
context = _resolve_context(args.context)
|
||||
|
||||
# Parse tags
|
||||
tags: list[str] = []
|
||||
if args.tags:
|
||||
tags = [t.strip() for t in args.tags.split(",") if t.strip()]
|
||||
|
||||
context_path = root / context / "knowledge"
|
||||
registry = LinkRegistry(context_path)
|
||||
|
||||
try:
|
||||
result_path = registry.save_link(
|
||||
url=args.url,
|
||||
title=args.title,
|
||||
tags=tags,
|
||||
)
|
||||
print(f"✓ Link gespeichert: {result_path}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"Fehler beim Speichern des Links: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main Entry Point (standalone usage)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def knowledge_main(args: argparse.Namespace) -> None:
|
||||
"""Main entry point for standalone knowledge CLI usage.
|
||||
|
||||
Dispatches to the appropriate handler based on knowledge_command.
|
||||
Calls sys.exit when no command is given.
|
||||
|
||||
Args:
|
||||
args: Parsed argparse namespace.
|
||||
"""
|
||||
knowledge_cmd = getattr(args, "knowledge_command", None)
|
||||
|
||||
if not knowledge_cmd:
|
||||
print(_HELP_TEXT)
|
||||
sys.exit(0)
|
||||
|
||||
dispatch = {
|
||||
"capture": _cmd_capture,
|
||||
"ingest": _cmd_ingest,
|
||||
"search": _cmd_search,
|
||||
"status": _cmd_status,
|
||||
"link": _cmd_link,
|
||||
}
|
||||
|
||||
handler = dispatch.get(knowledge_cmd)
|
||||
if handler is None:
|
||||
print(f"Unbekanntes Unterkommando: {knowledge_cmd}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
exit_code = handler(args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration for main CLI integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register_knowledge_subcommand(subparsers: argparse._SubParsersAction) -> None:
|
||||
"""Register the `knowledge` subcommand group under the main CLI parser.
|
||||
|
||||
This reuses the same parser structure as build_knowledge_parser but
|
||||
integrates it as a subparser of the main monorepo CLI.
|
||||
|
||||
Args:
|
||||
subparsers: The subparsers action from the main argparse parser.
|
||||
"""
|
||||
knowledge_parser = subparsers.add_parser(
|
||||
"knowledge",
|
||||
help="Wissensmanagement: Erfassen, Verarbeiten, Suchen",
|
||||
description=_HELP_TEXT,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
# Sub-subparsers for knowledge commands
|
||||
knowledge_sub = knowledge_parser.add_subparsers(
|
||||
dest="knowledge_command",
|
||||
help="Knowledge-Unterkommandos",
|
||||
)
|
||||
|
||||
# --- knowledge capture ---
|
||||
p_capture = knowledge_sub.add_parser(
|
||||
"capture",
|
||||
help="Text, URL oder Dateipfad schnell erfassen",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"input",
|
||||
help="Zu erfassender Text, URL oder Dateipfad.",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"--tags", "-t",
|
||||
default=None,
|
||||
help="Komma-getrennte Tags (z.B. 'api,meeting').",
|
||||
)
|
||||
|
||||
# --- knowledge ingest ---
|
||||
p_ingest = knowledge_sub.add_parser(
|
||||
"ingest",
|
||||
help="Ingestion-Pipeline für konfigurierte Quellen starten",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Vorschau ohne Schreiboperationen.",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--verbose", "-v",
|
||||
action="store_true",
|
||||
help="Detaillierte Ausgabe.",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--no-commit",
|
||||
action="store_true",
|
||||
help="Git-Commit überspringen.",
|
||||
)
|
||||
|
||||
# --- knowledge search ---
|
||||
p_search = knowledge_sub.add_parser(
|
||||
"search",
|
||||
help="Knowledge-Index durchsuchen",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"query",
|
||||
help="Suchbegriff.",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Suche auf einen Kontext beschränken.",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--limit", "-l",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Maximale Anzahl Ergebnisse (Standard: 20).",
|
||||
)
|
||||
|
||||
# --- knowledge status ---
|
||||
p_status = knowledge_sub.add_parser(
|
||||
"status",
|
||||
help="Übersicht über Inbox-Elemente und Artefakte",
|
||||
)
|
||||
p_status.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Kontext anzeigen (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
|
||||
# --- knowledge link ---
|
||||
p_link = knowledge_sub.add_parser(
|
||||
"link",
|
||||
help="Web-Link als Referenzmaterial speichern",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"url",
|
||||
help="Die zu speichernde URL.",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--title", "-T",
|
||||
default=None,
|
||||
help="Optionaler Titel für den Link.",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--tags", "-t",
|
||||
default=None,
|
||||
help="Komma-getrennte Tags (z.B. 'api,referenz').",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
|
||||
|
||||
def handle_knowledge_command(args: argparse.Namespace, root: Path) -> int:
|
||||
"""Dispatch knowledge subcommands from the main CLI.
|
||||
|
||||
Called from the main CLI's main() when 'knowledge' is the active command.
|
||||
Uses the root provided by the main CLI instead of discovering it again.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments namespace.
|
||||
root: Monorepo root path (provided by main CLI).
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, non-zero for failure).
|
||||
"""
|
||||
knowledge_cmd = getattr(args, "knowledge_command", None)
|
||||
|
||||
if not knowledge_cmd:
|
||||
print(_HELP_TEXT)
|
||||
return 0
|
||||
|
||||
dispatch = {
|
||||
"capture": _cmd_capture,
|
||||
"ingest": _cmd_ingest,
|
||||
"search": _cmd_search,
|
||||
"status": _cmd_status,
|
||||
"link": _cmd_link,
|
||||
}
|
||||
|
||||
handler = dispatch.get(knowledge_cmd)
|
||||
if handler is None:
|
||||
print(f"Unbekanntes Unterkommando: {knowledge_cmd}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
return handler(args)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Unified Ingestion Pipeline für den Wissensspeicher.
|
||||
|
||||
Orchestriert das Erfassen, Routen, Anreichern und Speichern von
|
||||
Wissensartefakten aus verschiedenen Quellen in die Knowledge-Kontexte.
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
from monorepo.knowledge.ingestion.config import (
|
||||
ConfigValidationError,
|
||||
EnrichmentSettings,
|
||||
GitSettings,
|
||||
OCRSettings,
|
||||
PipelineSettings,
|
||||
SourceConfig,
|
||||
load_sources_config,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.enrichment import (
|
||||
ActionItem,
|
||||
EnrichmentAgent,
|
||||
EnrichmentResult,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.git_integration import (
|
||||
auto_commit,
|
||||
auto_commit_capture,
|
||||
generate_capture_commit_message,
|
||||
generate_commit_message,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.pipeline import (
|
||||
IngestError,
|
||||
IngestionPipeline,
|
||||
IngestionResult,
|
||||
get_source_strategy,
|
||||
register_source_strategy,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.router import ContextRouter, RoutingDecision
|
||||
|
||||
__all__ = [
|
||||
"ActionItem",
|
||||
"ConfigValidationError",
|
||||
"ContextRouter",
|
||||
"EnrichmentAgent",
|
||||
"EnrichmentResult",
|
||||
"EnrichmentSettings",
|
||||
"GitSettings",
|
||||
"IngestError",
|
||||
"IngestionPipeline",
|
||||
"IngestionResult",
|
||||
"OCRSettings",
|
||||
"PipelineSettings",
|
||||
"QuickCapture",
|
||||
"RoutingDecision",
|
||||
"SourceConfig",
|
||||
"auto_commit",
|
||||
"auto_commit_capture",
|
||||
"generate_capture_commit_message",
|
||||
"generate_commit_message",
|
||||
"get_source_strategy",
|
||||
"load_sources_config",
|
||||
"register_source_strategy",
|
||||
]
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Capture – Schnelle Wissenserfassung über CLI und Integrationen.
|
||||
|
||||
Ermöglicht das ad-hoc Erfassen von Wissensschnipseln, Notizen und
|
||||
Erkenntnissen ohne vollständigen Pipeline-Durchlauf.
|
||||
|
||||
Requirements: 3.1, 3.2, 3.3, 3.7, 8.3
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from monorepo.knowledge.ingestion.router import ContextRouter
|
||||
|
||||
|
||||
class QuickCapture:
|
||||
"""Fast knowledge capture to inbox."""
|
||||
|
||||
def __init__(self, monorepo_root: Path) -> None:
|
||||
"""Initialize with monorepo root path.
|
||||
|
||||
Args:
|
||||
monorepo_root: Absolute path to the monorepo root directory.
|
||||
"""
|
||||
self.monorepo_root = monorepo_root
|
||||
self._router = ContextRouter(monorepo_root)
|
||||
|
||||
def capture(
|
||||
self,
|
||||
input_text: str,
|
||||
context: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
) -> Path:
|
||||
"""Capture text/URL/filepath to inbox. Returns path of created artifact.
|
||||
|
||||
Args:
|
||||
input_text: The text, URL, or file path to capture.
|
||||
context: Target context (bahn, dhive, privat). If None,
|
||||
derived from current working directory.
|
||||
tags: Optional list of tags.
|
||||
|
||||
Returns:
|
||||
Path of the created Markdown artifact.
|
||||
|
||||
Raises:
|
||||
ValueError: If context cannot be determined.
|
||||
"""
|
||||
if tags is None:
|
||||
tags = []
|
||||
|
||||
# Determine context
|
||||
if context is None:
|
||||
context = self._router.determine_context(cwd=Path.cwd())
|
||||
|
||||
# Detect input type
|
||||
input_type = self._detect_input_type(input_text)
|
||||
|
||||
# Generate title based on input type
|
||||
title = self._derive_title(input_text, input_type)
|
||||
|
||||
# Generate filename
|
||||
filename = self._generate_filename(title)
|
||||
|
||||
# Build target path: {context}/knowledge/inbox/{filename}
|
||||
target_dir = self.monorepo_root / context / "knowledge" / "inbox"
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
target_path = target_dir / filename
|
||||
|
||||
# Build frontmatter
|
||||
now = datetime.now()
|
||||
frontmatter = self._build_frontmatter(
|
||||
title=title,
|
||||
created=now.strftime("%Y-%m-%d"),
|
||||
source_type=input_type,
|
||||
source_value=input_text,
|
||||
tags=tags,
|
||||
source_context=context,
|
||||
)
|
||||
|
||||
# Build content body
|
||||
body = self._build_body(input_text, input_type)
|
||||
|
||||
# Write file
|
||||
content = f"{frontmatter}\n{body}\n"
|
||||
target_path.write_text(content, encoding="utf-8")
|
||||
|
||||
return target_path
|
||||
|
||||
def _detect_input_type(self, input_text: str) -> str:
|
||||
"""Detect if input is URL, file path, or plain text.
|
||||
|
||||
Args:
|
||||
input_text: The raw input string.
|
||||
|
||||
Returns:
|
||||
One of 'url', 'file', or 'text'.
|
||||
"""
|
||||
stripped = input_text.strip()
|
||||
|
||||
# Check for URL
|
||||
if stripped.startswith("http://") or stripped.startswith("https://"):
|
||||
return "url"
|
||||
|
||||
# Check for file path
|
||||
if self._looks_like_filepath(stripped):
|
||||
return "file"
|
||||
|
||||
return "text"
|
||||
|
||||
def _looks_like_filepath(self, text: str) -> bool:
|
||||
"""Heuristic: does the text look like a file path?
|
||||
|
||||
Returns True if:
|
||||
- It contains path separators AND has a file extension, OR
|
||||
- Path(text).exists()
|
||||
"""
|
||||
# Contains separators and has an extension
|
||||
has_separator = "/" in text or "\\" in text
|
||||
has_extension = bool(re.search(r"\.\w{1,10}$", text))
|
||||
|
||||
if has_separator and has_extension:
|
||||
return True
|
||||
|
||||
# Actually exists on disk
|
||||
try:
|
||||
return Path(text).exists()
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
def _derive_title(self, input_text: str, input_type: str) -> str:
|
||||
"""Derive a title from the input based on its type.
|
||||
|
||||
Args:
|
||||
input_text: The raw input.
|
||||
input_type: One of 'url', 'file', 'text'.
|
||||
|
||||
Returns:
|
||||
A human-readable title string.
|
||||
"""
|
||||
stripped = input_text.strip()
|
||||
|
||||
if input_type == "url":
|
||||
parsed = urlparse(stripped)
|
||||
# domain + path, strip leading/trailing slashes
|
||||
path_part = parsed.path.strip("/")
|
||||
if path_part:
|
||||
return f"{parsed.netloc}/{path_part}"
|
||||
return parsed.netloc
|
||||
|
||||
if input_type == "file":
|
||||
return Path(stripped).stem
|
||||
|
||||
# Text: first 60 characters
|
||||
first_line = stripped.split("\n", 1)[0]
|
||||
return first_line[:60]
|
||||
|
||||
def _generate_filename(self, title: str) -> str:
|
||||
"""Generate YYYY-MM-DD-HH-MM-slugified-title.md filename.
|
||||
|
||||
Args:
|
||||
title: The title to slugify for the filename.
|
||||
|
||||
Returns:
|
||||
A filename string with timestamp prefix and slugified title.
|
||||
"""
|
||||
now = datetime.now()
|
||||
timestamp = now.strftime("%Y-%m-%d-%H-%M")
|
||||
|
||||
slug = self._slugify(title)
|
||||
# Limit slug to 50 chars
|
||||
slug = slug[:50].rstrip("-")
|
||||
|
||||
# Fallback if slugification removed all characters (e.g. non-ASCII-only titles)
|
||||
if not slug:
|
||||
slug = "untitled"
|
||||
|
||||
return f"{timestamp}-{slug}.md"
|
||||
|
||||
def _slugify(self, text: str) -> str:
|
||||
"""Convert text to URL-safe slug.
|
||||
|
||||
Lowercase, replace non-alphanumeric with hyphens,
|
||||
collapse multiple hyphens, strip leading/trailing hyphens.
|
||||
"""
|
||||
slug = text.lower()
|
||||
# Replace non-alphanumeric (keep hyphens) with hyphens
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
||||
# Collapse multiple hyphens
|
||||
slug = re.sub(r"-{2,}", "-", slug)
|
||||
# Strip leading/trailing hyphens
|
||||
slug = slug.strip("-")
|
||||
return slug
|
||||
|
||||
def _build_frontmatter(
|
||||
self,
|
||||
title: str,
|
||||
created: str,
|
||||
source_type: str,
|
||||
source_value: str,
|
||||
tags: list[str],
|
||||
source_context: str,
|
||||
) -> str:
|
||||
"""Build YAML frontmatter block.
|
||||
|
||||
Args:
|
||||
title: Artifact title.
|
||||
created: Creation date (YYYY-MM-DD).
|
||||
source_type: Type of source (url, file, text).
|
||||
source_value: Original input value.
|
||||
tags: List of tags.
|
||||
source_context: The target context.
|
||||
|
||||
Returns:
|
||||
Complete YAML frontmatter as string including delimiters.
|
||||
"""
|
||||
# Escape title for YAML (wrap in quotes if it contains special chars)
|
||||
escaped_title = title.replace('"', '\\"')
|
||||
|
||||
lines = [
|
||||
"---",
|
||||
"type: inbox",
|
||||
f'title: "{escaped_title}"',
|
||||
f"created: {created}",
|
||||
"source:",
|
||||
f" type: {source_type}",
|
||||
f' value: "{source_value}"',
|
||||
]
|
||||
|
||||
if tags:
|
||||
tags_str = ", ".join(tags)
|
||||
lines.append(f"tags: [{tags_str}]")
|
||||
else:
|
||||
lines.append("tags: []")
|
||||
|
||||
lines.append(f"source_context: {source_context}")
|
||||
lines.append("---")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _build_body(self, input_text: str, input_type: str) -> str:
|
||||
"""Build the Markdown content body.
|
||||
|
||||
Args:
|
||||
input_text: The raw input.
|
||||
input_type: One of 'url', 'file', 'text'.
|
||||
|
||||
Returns:
|
||||
Markdown body content.
|
||||
"""
|
||||
stripped = input_text.strip()
|
||||
|
||||
if input_type == "url":
|
||||
return f"\n[{stripped}]({stripped})\n"
|
||||
|
||||
if input_type == "file":
|
||||
return f"\nSource file: `{stripped}`\n"
|
||||
|
||||
# Plain text
|
||||
return f"\n{stripped}\n"
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Config – Konfigurationsmodell für die Ingestion-Pipeline.
|
||||
|
||||
Definiert das Schema für sources.yaml und Pipeline-Einstellungen
|
||||
pro Kontext. Unterstützt Umgebungsvariablen-Auflösung (${VAR_NAME})
|
||||
und Validierung der Konfiguration vor Pipeline-Start.
|
||||
|
||||
Requirements: 12.1, 12.2, 12.4, 12.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ConfigValidationError(Exception):
|
||||
"""Wird ausgelöst bei ungültiger sources.yaml-Konfiguration.
|
||||
|
||||
Enthält eine Liste aller gefundenen Validierungsfehler.
|
||||
"""
|
||||
|
||||
def __init__(self, errors: list[str]) -> None:
|
||||
self.errors = errors
|
||||
msg = "Konfigurationsvalidierung fehlgeschlagen:\n" + "\n".join(
|
||||
f" - {e}" for e in errors
|
||||
)
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Datenmodelle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceConfig:
|
||||
"""Configuration for a single source in sources.yaml."""
|
||||
|
||||
type: str # confluence, jira, email, file, link, wissensdatenbank
|
||||
name: str # human-readable identifier
|
||||
params: dict[str, Any] = field(default_factory=dict) # type-specific params
|
||||
target_folder: str = "" # override target subfolder
|
||||
sync_frequency: str = "manual" # on-push, hourly, daily, manual
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichmentSettings:
|
||||
"""Einstellungen für den Enrichment-Agenten."""
|
||||
|
||||
provider: str = "kiro"
|
||||
model: str = ""
|
||||
confidence_threshold: float = 0.7
|
||||
|
||||
|
||||
@dataclass
|
||||
class OCRSettings:
|
||||
"""Einstellungen für die OCR-Engine."""
|
||||
|
||||
provider: str = "kiro"
|
||||
languages: str = "deu+eng"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GitSettings:
|
||||
"""Einstellungen für die Git-Integration."""
|
||||
|
||||
auto_commit: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineSettings:
|
||||
"""Pipeline-Einstellungen aus dem settings-Block in sources.yaml."""
|
||||
|
||||
create_tasks: bool = True
|
||||
enrichment: EnrichmentSettings = field(default_factory=EnrichmentSettings)
|
||||
ocr: OCRSettings = field(default_factory=OCRSettings)
|
||||
git: GitSettings = field(default_factory=GitSettings)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Umgebungsvariablen-Auflösung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_ENV_VAR_PATTERN = re.compile(r"\$\{([^}]+)\}")
|
||||
|
||||
|
||||
def resolve_env_vars(value: str) -> str:
|
||||
"""Ersetzt alle ${VAR_NAME}-Patterns in einem String durch Umgebungsvariablen.
|
||||
|
||||
Args:
|
||||
value: Der Eingabestring mit möglichen ${VAR_NAME}-Patterns.
|
||||
|
||||
Returns:
|
||||
Der String mit aufgelösten Umgebungsvariablen.
|
||||
|
||||
Raises:
|
||||
ConfigValidationError: Wenn eine referenzierte Variable nicht gesetzt ist.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
def _replace(match: re.Match[str]) -> str:
|
||||
var_name = match.group(1)
|
||||
env_value = os.environ.get(var_name)
|
||||
if env_value is None:
|
||||
errors.append(
|
||||
f"Umgebungsvariable '${{{var_name}}}' ist nicht gesetzt"
|
||||
)
|
||||
return match.group(0) # Originaltext beibehalten für Fehlermeldung
|
||||
return env_value
|
||||
|
||||
result = _ENV_VAR_PATTERN.sub(_replace, value)
|
||||
|
||||
if errors:
|
||||
raise ConfigValidationError(errors)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_env_vars_recursive(obj: Any) -> Any:
|
||||
"""Löst ${VAR_NAME}-Patterns rekursiv in verschachtelten Datenstrukturen auf.
|
||||
|
||||
Traversiert dicts, Listen und Strings. Nicht-String-Werte werden
|
||||
unverändert zurückgegeben.
|
||||
|
||||
Args:
|
||||
obj: Die zu traversierende Datenstruktur.
|
||||
|
||||
Returns:
|
||||
Die Datenstruktur mit aufgelösten Umgebungsvariablen.
|
||||
|
||||
Raises:
|
||||
ConfigValidationError: Wenn Variablen nicht aufgelöst werden können.
|
||||
"""
|
||||
if isinstance(obj, str):
|
||||
return resolve_env_vars(obj)
|
||||
elif isinstance(obj, dict):
|
||||
return {k: _resolve_env_vars_recursive(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, list):
|
||||
return [_resolve_env_vars_recursive(item) for item in obj]
|
||||
return obj
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validierung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _validate_source_entry(entry: dict[str, Any], index: int) -> list[str]:
|
||||
"""Validiert einen einzelnen Source-Eintrag.
|
||||
|
||||
Args:
|
||||
entry: Der rohe dict-Eintrag aus der YAML-Datei.
|
||||
index: Der Index in der sources-Liste (für Fehlermeldungen).
|
||||
|
||||
Returns:
|
||||
Liste von Validierungsfehlern (leer wenn gültig).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
if not isinstance(entry, dict):
|
||||
errors.append(f"Source #{index + 1}: Eintrag muss ein Mapping sein")
|
||||
return errors
|
||||
|
||||
# Pflichtfelder prüfen
|
||||
if not entry.get("type"):
|
||||
errors.append(f"Source #{index + 1}: Pflichtfeld 'type' fehlt oder ist leer")
|
||||
if not entry.get("name"):
|
||||
errors.append(f"Source #{index + 1}: Pflichtfeld 'name' fehlt oder ist leer")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _collect_env_var_errors(obj: Any, path: str = "") -> list[str]:
|
||||
"""Sammelt alle nicht auflösbaren Umgebungsvariablen ohne Exception.
|
||||
|
||||
Traversiert die Datenstruktur und identifiziert alle ${VAR_NAME}-Patterns,
|
||||
deren Variablen nicht gesetzt sind.
|
||||
|
||||
Args:
|
||||
obj: Die zu prüfende Datenstruktur.
|
||||
path: Der aktuelle Pfad für Fehlermeldungen.
|
||||
|
||||
Returns:
|
||||
Liste von Fehlermeldungen für nicht-auflösbare Variablen.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
if isinstance(obj, str):
|
||||
for match in _ENV_VAR_PATTERN.finditer(obj):
|
||||
var_name = match.group(1)
|
||||
if os.environ.get(var_name) is None:
|
||||
location = f" (in {path})" if path else ""
|
||||
errors.append(
|
||||
f"Umgebungsvariable '${{{var_name}}}' ist nicht gesetzt{location}"
|
||||
)
|
||||
elif isinstance(obj, dict):
|
||||
for key, value in obj.items():
|
||||
sub_path = f"{path}.{key}" if path else key
|
||||
errors.extend(_collect_env_var_errors(value, sub_path))
|
||||
elif isinstance(obj, list):
|
||||
for i, item in enumerate(obj):
|
||||
sub_path = f"{path}[{i}]"
|
||||
errors.extend(_collect_env_var_errors(item, sub_path))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sources.yaml Laden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_sources_config(path: Path) -> tuple[list[SourceConfig], PipelineSettings]:
|
||||
"""Parst eine sources.yaml-Datei und gibt Quellkonfigurationen + Settings zurück.
|
||||
|
||||
Ablauf:
|
||||
1. YAML-Datei einlesen
|
||||
2. Pflichtfelder validieren
|
||||
3. Umgebungsvariablen in allen String-Werten auflösen
|
||||
4. SourceConfig- und PipelineSettings-Objekte erstellen
|
||||
|
||||
Args:
|
||||
path: Pfad zur sources.yaml-Datei.
|
||||
|
||||
Returns:
|
||||
Tuple aus (Liste von SourceConfig, PipelineSettings).
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn die Datei nicht existiert.
|
||||
ConfigValidationError: Bei Validierungsfehlern (fehlende Felder,
|
||||
nicht-auflösbare Umgebungsvariablen).
|
||||
"""
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Konfigurationsdatei nicht gefunden: {path}")
|
||||
|
||||
text = path.read_text(encoding="utf-8")
|
||||
raw: dict[str, Any] = yaml.safe_load(text) or {}
|
||||
|
||||
# --- Phase 1: Strukturvalidierung ---
|
||||
validation_errors: list[str] = []
|
||||
|
||||
sources_raw = raw.get("sources", [])
|
||||
if not isinstance(sources_raw, list):
|
||||
validation_errors.append("'sources' muss eine Liste sein")
|
||||
sources_raw = []
|
||||
|
||||
for i, entry in enumerate(sources_raw):
|
||||
validation_errors.extend(_validate_source_entry(entry, i))
|
||||
|
||||
# --- Phase 2: Env-Var-Validierung (alle sammeln, nicht beim ersten abbrechen) ---
|
||||
for i, entry in enumerate(sources_raw):
|
||||
if isinstance(entry, dict):
|
||||
env_errors = _collect_env_var_errors(
|
||||
entry.get("params", {}), f"sources[{i}].params"
|
||||
)
|
||||
validation_errors.extend(env_errors)
|
||||
|
||||
# Settings-Block auch auf env vars prüfen
|
||||
settings_raw = raw.get("settings", {})
|
||||
if isinstance(settings_raw, dict):
|
||||
env_errors = _collect_env_var_errors(settings_raw, "settings")
|
||||
validation_errors.extend(env_errors)
|
||||
|
||||
if validation_errors:
|
||||
raise ConfigValidationError(validation_errors)
|
||||
|
||||
# --- Phase 3: Env-Vars auflösen und Objekte erstellen ---
|
||||
sources: list[SourceConfig] = []
|
||||
for entry in sources_raw:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
|
||||
# params mit aufgelösten env vars
|
||||
resolved_params = _resolve_env_vars_recursive(entry.get("params", {}))
|
||||
|
||||
sources.append(
|
||||
SourceConfig(
|
||||
type=str(entry.get("type", "")),
|
||||
name=str(entry.get("name", "")),
|
||||
params=resolved_params if isinstance(resolved_params, dict) else {},
|
||||
target_folder=str(entry.get("target_folder", "")),
|
||||
sync_frequency=str(entry.get("sync_frequency", "manual")),
|
||||
enabled=bool(entry.get("enabled", True)),
|
||||
)
|
||||
)
|
||||
|
||||
# --- Phase 4: PipelineSettings parsen ---
|
||||
settings = _parse_pipeline_settings(settings_raw)
|
||||
|
||||
return sources, settings
|
||||
|
||||
|
||||
def _parse_pipeline_settings(raw: Any) -> PipelineSettings:
|
||||
"""Parst den settings-Block aus sources.yaml.
|
||||
|
||||
Args:
|
||||
raw: Der rohe dict aus dem YAML.
|
||||
|
||||
Returns:
|
||||
Ein PipelineSettings-Objekt mit Defaults für fehlende Felder.
|
||||
"""
|
||||
if not isinstance(raw, dict):
|
||||
return PipelineSettings()
|
||||
|
||||
# Enrichment settings
|
||||
enrichment_raw = raw.get("enrichment", {})
|
||||
enrichment = EnrichmentSettings()
|
||||
if isinstance(enrichment_raw, dict):
|
||||
# Env vars im enrichment-Block auflösen
|
||||
resolved_enrichment = _resolve_env_vars_recursive(enrichment_raw)
|
||||
if isinstance(resolved_enrichment, dict):
|
||||
enrichment = EnrichmentSettings(
|
||||
provider=str(resolved_enrichment.get("provider", "kiro")),
|
||||
model=str(resolved_enrichment.get("model", "")),
|
||||
confidence_threshold=float(
|
||||
resolved_enrichment.get("confidence_threshold", 0.7)
|
||||
),
|
||||
)
|
||||
|
||||
# OCR settings
|
||||
ocr_raw = raw.get("ocr", {})
|
||||
ocr = OCRSettings()
|
||||
if isinstance(ocr_raw, dict):
|
||||
resolved_ocr = _resolve_env_vars_recursive(ocr_raw)
|
||||
if isinstance(resolved_ocr, dict):
|
||||
ocr = OCRSettings(
|
||||
provider=str(resolved_ocr.get("provider", "kiro")),
|
||||
languages=str(resolved_ocr.get("languages", "deu+eng")),
|
||||
)
|
||||
|
||||
# Git settings
|
||||
git_raw = raw.get("git", {})
|
||||
git = GitSettings()
|
||||
if isinstance(git_raw, dict):
|
||||
git = GitSettings(
|
||||
auto_commit=bool(git_raw.get("auto_commit", True)),
|
||||
)
|
||||
|
||||
return PipelineSettings(
|
||||
create_tasks=bool(raw.get("create_tasks", True)),
|
||||
enrichment=enrichment,
|
||||
ocr=ocr,
|
||||
git=git,
|
||||
)
|
||||
@@ -0,0 +1,692 @@
|
||||
"""Enrichment – KI-gestützte Anreicherung von Wissensartefakten.
|
||||
|
||||
Extrahiert Tags, Kategorien, Entitäten und Action-Items aus
|
||||
Rohartefakten mittels LLM-Aufrufen (Kiro-first) oder regelbasierten
|
||||
Heuristiken als Fallback.
|
||||
|
||||
Requirements: 9.1, 9.2, 9.3, 9.4, 9.5, 9.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionItem:
|
||||
"""An extracted action item from content.
|
||||
|
||||
Attributes:
|
||||
description: The action item text.
|
||||
assignee: Optional person assigned to the action.
|
||||
deadline: Optional ISO date string for the deadline.
|
||||
hash: SHA-256 hash of description for deduplication.
|
||||
"""
|
||||
|
||||
description: str
|
||||
assignee: str | None = None
|
||||
deadline: str | None = None # ISO date string
|
||||
hash: str = "" # for dedup: sha256 of description
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.hash and self.description:
|
||||
self.hash = _compute_action_item_hash(self.description)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichmentResult:
|
||||
"""Output from enrichment analysis.
|
||||
|
||||
Attributes:
|
||||
title: Extracted or generated title.
|
||||
category: Content category (meeting, decision, project, reference, link, inbox).
|
||||
tags: Generated tags describing the content.
|
||||
people: Detected person names (confidence >= 0.7).
|
||||
projects: Detected project references (confidence >= 0.7).
|
||||
action_items: Extracted action items.
|
||||
confidence: Overall confidence score (0.0 - 1.0).
|
||||
enrichment_pending: True if LLM provider was unavailable.
|
||||
"""
|
||||
|
||||
title: str
|
||||
category: str # meeting, decision, project, reference, link, inbox
|
||||
tags: list[str] = field(default_factory=list)
|
||||
people: list[str] = field(default_factory=list)
|
||||
projects: list[str] = field(default_factory=list)
|
||||
action_items: list[ActionItem] = field(default_factory=list)
|
||||
confidence: float = 0.0
|
||||
enrichment_pending: bool = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Minimum confidence threshold for including entities in output.
|
||||
CONFIDENCE_THRESHOLD = 0.7
|
||||
|
||||
#: Valid categories for artifact classification.
|
||||
VALID_CATEGORIES = ("meeting", "decision", "project", "reference", "link", "inbox")
|
||||
|
||||
#: Category detection keywords (German + English).
|
||||
CATEGORY_KEYWORDS: dict[str, list[str]] = {
|
||||
"meeting": [
|
||||
"meeting", "besprechung", "jour fixe", "standup", "stand-up",
|
||||
"sync", "retro", "retrospektive", "daily", "weekly",
|
||||
"protokoll", "minutes", "teilnehmer", "participants",
|
||||
"agenda", "termin",
|
||||
],
|
||||
"decision": [
|
||||
"decision", "entscheidung", "beschluss", "agreed", "decided",
|
||||
"resolution", "entschieden", "festgelegt", "approval",
|
||||
"genehmigung", "freigabe",
|
||||
],
|
||||
"project": [
|
||||
"projekt", "project", "milestone", "meilenstein", "sprint",
|
||||
"release", "roadmap", "backlog", "epic", "feature",
|
||||
"workstream", "initiative",
|
||||
],
|
||||
"reference": [
|
||||
"referenz", "reference", "dokumentation", "documentation",
|
||||
"howto", "how-to", "anleitung", "tutorial", "guide",
|
||||
"handbuch", "manual", "wiki", "spec", "specification",
|
||||
],
|
||||
"link": [
|
||||
"http://", "https://", "www.",
|
||||
],
|
||||
}
|
||||
|
||||
#: Patterns for detecting action items in text.
|
||||
ACTION_ITEM_PATTERNS: list[re.Pattern[str]] = [
|
||||
re.compile(r"^[-*]\s*\[\s*\]\s*(.+)", re.MULTILINE), # - [ ] task
|
||||
re.compile(r"^TODO:\s*(.+)", re.MULTILINE | re.IGNORECASE), # TODO: task
|
||||
re.compile(r"^ACTION:\s*(.+)", re.MULTILINE | re.IGNORECASE), # ACTION: task
|
||||
re.compile(r"^→\s*(.+)", re.MULTILINE), # → task
|
||||
re.compile(r"^=>\s*(.+)", re.MULTILINE), # => task
|
||||
]
|
||||
|
||||
#: Pattern for detecting person names (Vorname Nachname).
|
||||
# Supports Latin letters with diacritics (é, è, ñ, etc.).
|
||||
PERSON_NAME_PATTERN = re.compile(
|
||||
r"(?<![#/\[])(?:^|\s)(@?[A-ZÀ-ÖØ-Þ][a-zà-öø-ÿß]{2,}\s[A-ZÀ-ÖØ-Þ][a-zà-öø-ÿß]{2,})(?:\s|[,;.\-!?]|$)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
#: Pattern for @mentions.
|
||||
AT_MENTION_PATTERN = re.compile(r"@([A-Za-zÄÖÜäöüß]+(?:\s[A-Za-zÄÖÜäöüß]+)?)")
|
||||
|
||||
#: Pattern for detecting project references (e.g., "Projekt XYZ", "Project: ABC").
|
||||
PROJECT_REF_PATTERNS: list[re.Pattern[str]] = [
|
||||
re.compile(r"(?:Projekt|Project)\s*[:\-]?\s*([A-Za-zÄÖÜäöüß0-9][\w\s\-]{2,30})", re.IGNORECASE),
|
||||
re.compile(r"\[\[projects/([^\]]+)\]\]"), # existing wiki links
|
||||
]
|
||||
|
||||
#: ISO date pattern for deadline detection.
|
||||
ISO_DATE_PATTERN = re.compile(
|
||||
r"(?:bis|by|until|deadline:?|fällig:?)\s*(\d{4}-\d{2}-\d{2})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
#: Common stop words to exclude from people detection.
|
||||
_STOP_NAMES: set[str] = {
|
||||
"Action Items", "Quick Capture", "Knowledge Management",
|
||||
"Meeting Notes", "Daily Standup", "Deutsche Bahn",
|
||||
"Sprint Review", "Sprint Planning", "Open Source",
|
||||
"Pull Request", "Merge Request", "Code Review",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper Functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slugify(name: str) -> str:
|
||||
"""Convert a name to a URL/wiki-link slug.
|
||||
|
||||
Converts "André Knie" → "andre-knie", "Max Müller" → "max-mueller".
|
||||
|
||||
Args:
|
||||
name: The name to slugify.
|
||||
|
||||
Returns:
|
||||
Lowercase hyphenated slug suitable for wiki-links.
|
||||
"""
|
||||
# Normalize unicode (decompose then remove combining marks)
|
||||
text = name.strip().lower()
|
||||
|
||||
# German umlaut replacements before NFD decomposition
|
||||
replacements = {
|
||||
"ä": "ae", "ö": "oe", "ü": "ue",
|
||||
"ß": "ss",
|
||||
}
|
||||
for char, repl in replacements.items():
|
||||
text = text.replace(char, repl)
|
||||
|
||||
# Remove remaining diacritics
|
||||
text = unicodedata.normalize("NFD", text)
|
||||
text = "".join(c for c in text if unicodedata.category(c) != "Mn")
|
||||
|
||||
# Replace non-alphanumeric with hyphens
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text)
|
||||
|
||||
# Collapse multiple hyphens and strip leading/trailing
|
||||
text = re.sub(r"-+", "-", text).strip("-")
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _compute_action_item_hash(description: str) -> str:
|
||||
"""Compute SHA-256 hash of an action item description for deduplication.
|
||||
|
||||
Args:
|
||||
description: The action item description text.
|
||||
|
||||
Returns:
|
||||
First 16 characters of hex SHA-256 digest.
|
||||
"""
|
||||
normalized = description.strip().lower()
|
||||
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _detect_category(content: str, source_type: str = "") -> tuple[str, float]:
|
||||
"""Detect the content category using keyword matching.
|
||||
|
||||
Args:
|
||||
content: The text content to classify.
|
||||
source_type: Optional source type hint (e.g., "email", "confluence").
|
||||
|
||||
Returns:
|
||||
Tuple of (category, confidence).
|
||||
"""
|
||||
content_lower = content.lower()
|
||||
|
||||
# Source type can give strong hints
|
||||
source_type_hints: dict[str, str] = {
|
||||
"email": "reference",
|
||||
"chat": "reference",
|
||||
"confluence": "reference",
|
||||
"jira": "project",
|
||||
}
|
||||
|
||||
scores: dict[str, float] = {cat: 0.0 for cat in VALID_CATEGORIES}
|
||||
|
||||
# Count keyword matches per category
|
||||
for category, keywords in CATEGORY_KEYWORDS.items():
|
||||
for keyword in keywords:
|
||||
if keyword in content_lower:
|
||||
scores[category] += 1.0
|
||||
|
||||
# Apply source type hint bonus
|
||||
if source_type and source_type.lower() in source_type_hints:
|
||||
hinted = source_type_hints[source_type.lower()]
|
||||
scores[hinted] += 0.5
|
||||
|
||||
# Check if content is primarily a URL/link
|
||||
stripped = content.strip()
|
||||
if stripped.startswith(("http://", "https://")) and "\n" not in stripped:
|
||||
return "link", 0.9
|
||||
|
||||
# Find best category
|
||||
best_category = max(scores, key=lambda k: scores[k])
|
||||
best_score = scores[best_category]
|
||||
|
||||
if best_score == 0.0:
|
||||
return "inbox", 0.3
|
||||
|
||||
# Normalize confidence (more matches = higher confidence, capped at 0.95)
|
||||
confidence = min(0.95, 0.5 + best_score * 0.1)
|
||||
|
||||
return best_category, confidence
|
||||
|
||||
|
||||
def _extract_people(content: str) -> list[tuple[str, float]]:
|
||||
"""Extract person names from content with confidence scores.
|
||||
|
||||
Detects patterns like "Max Müller", "@andre.knie", "@André Knie".
|
||||
|
||||
Args:
|
||||
content: The text to analyze.
|
||||
|
||||
Returns:
|
||||
List of (name, confidence) tuples.
|
||||
"""
|
||||
people: dict[str, float] = {}
|
||||
|
||||
# Detect @mentions (high confidence)
|
||||
for match in AT_MENTION_PATTERN.finditer(content):
|
||||
name = match.group(1).strip()
|
||||
if len(name) >= 3 and name not in _STOP_NAMES:
|
||||
people[name] = max(people.get(name, 0), 0.9)
|
||||
|
||||
# Detect "Vorname Nachname" patterns
|
||||
for match in PERSON_NAME_PATTERN.finditer(content):
|
||||
name = match.group(1).strip().lstrip("@")
|
||||
if name in _STOP_NAMES:
|
||||
continue
|
||||
if len(name) >= 5: # at least "Xx Yy" length
|
||||
# Lower confidence for bare name patterns (could be titles, places)
|
||||
people[name] = max(people.get(name, 0), 0.75)
|
||||
|
||||
return [(name, conf) for name, conf in people.items()]
|
||||
|
||||
|
||||
def _extract_projects(content: str) -> list[tuple[str, float]]:
|
||||
"""Extract project references from content with confidence scores.
|
||||
|
||||
Args:
|
||||
content: The text to analyze.
|
||||
|
||||
Returns:
|
||||
List of (project_name, confidence) tuples.
|
||||
"""
|
||||
projects: dict[str, float] = {}
|
||||
|
||||
for pattern in PROJECT_REF_PATTERNS:
|
||||
for match in pattern.finditer(content):
|
||||
proj = match.group(1).strip().rstrip(".,;:!?")
|
||||
if len(proj) >= 2:
|
||||
projects[proj] = max(projects.get(proj, 0), 0.8)
|
||||
|
||||
return [(name, conf) for name, conf in projects.items()]
|
||||
|
||||
|
||||
def _extract_action_items(content: str) -> list[ActionItem]:
|
||||
"""Extract action items from content.
|
||||
|
||||
Detects patterns like:
|
||||
- [ ] Do something
|
||||
TODO: Something else
|
||||
ACTION: Assign this task
|
||||
→ Next step
|
||||
|
||||
Args:
|
||||
content: The text to analyze.
|
||||
|
||||
Returns:
|
||||
List of ActionItem instances.
|
||||
"""
|
||||
seen_hashes: set[str] = set()
|
||||
items: list[ActionItem] = []
|
||||
|
||||
for pattern in ACTION_ITEM_PATTERNS:
|
||||
for match in pattern.finditer(content):
|
||||
description = match.group(1).strip()
|
||||
if not description or len(description) < 3:
|
||||
continue
|
||||
|
||||
item_hash = _compute_action_item_hash(description)
|
||||
if item_hash in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(item_hash)
|
||||
|
||||
# Try to extract assignee from description
|
||||
assignee = _extract_assignee(description)
|
||||
|
||||
# Try to extract deadline
|
||||
deadline = _extract_deadline(description, content)
|
||||
|
||||
items.append(ActionItem(
|
||||
description=description,
|
||||
assignee=assignee,
|
||||
deadline=deadline,
|
||||
hash=item_hash,
|
||||
))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _extract_assignee(description: str) -> str | None:
|
||||
"""Try to extract an assignee from action item text.
|
||||
|
||||
Looks for patterns like "@Name" or "(Name)" at the end.
|
||||
|
||||
Args:
|
||||
description: The action item description.
|
||||
|
||||
Returns:
|
||||
Assignee name or None.
|
||||
"""
|
||||
# Check for @mention in description (supports diacritics)
|
||||
at_match = re.search(r"@([\w\u00C0-\u024F]+(?:\s[\w\u00C0-\u024F]+)?)", description)
|
||||
if at_match:
|
||||
return at_match.group(1)
|
||||
|
||||
# Check for (Name) at end
|
||||
paren_match = re.search(
|
||||
r"\(([A-ZÀ-ÖØ-Þ][\w\u00C0-\u024F]+(?:\s[A-ZÀ-ÖØ-Þ][\w\u00C0-\u024F]+)?)\)\s*$",
|
||||
description,
|
||||
)
|
||||
if paren_match:
|
||||
return paren_match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_deadline(description: str, full_content: str = "") -> str | None:
|
||||
"""Try to extract a deadline from action item or surrounding context.
|
||||
|
||||
Args:
|
||||
description: The action item description.
|
||||
full_content: The full document content for context.
|
||||
|
||||
Returns:
|
||||
ISO date string or None.
|
||||
"""
|
||||
# Look in description first
|
||||
date_match = ISO_DATE_PATTERN.search(description)
|
||||
if date_match:
|
||||
return date_match.group(1)
|
||||
|
||||
# Simple date pattern in description
|
||||
simple_date = re.search(r"(\d{4}-\d{2}-\d{2})", description)
|
||||
if simple_date:
|
||||
return simple_date.group(1)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_title(content: str) -> str:
|
||||
"""Extract title from first heading or first line of content.
|
||||
|
||||
Args:
|
||||
content: The text content.
|
||||
|
||||
Returns:
|
||||
Extracted title string.
|
||||
"""
|
||||
lines = content.strip().split("\n")
|
||||
|
||||
for line in lines[:5]: # Check first 5 lines
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Markdown heading
|
||||
heading_match = re.match(r"^#{1,3}\s+(.+)", line)
|
||||
if heading_match:
|
||||
return heading_match.group(1).strip()
|
||||
|
||||
# First non-empty line as fallback title
|
||||
if len(line) > 2:
|
||||
# Truncate very long first lines
|
||||
return line[:80].rstrip(".,;:!?")
|
||||
|
||||
return "Untitled"
|
||||
|
||||
|
||||
def _extract_tags(content: str, category: str) -> list[str]:
|
||||
"""Extract relevant tags from content using keyword frequency.
|
||||
|
||||
Args:
|
||||
content: The text content to analyze.
|
||||
category: The detected category (used as base tag).
|
||||
|
||||
Returns:
|
||||
List of tag strings.
|
||||
"""
|
||||
tags: list[str] = []
|
||||
|
||||
# Add category as a tag if meaningful
|
||||
if category and category != "inbox":
|
||||
tags.append(category)
|
||||
|
||||
content_lower = content.lower()
|
||||
|
||||
# Domain-specific tag detection
|
||||
tech_keywords = {
|
||||
"python", "java", "typescript", "docker", "kubernetes", "k8s",
|
||||
"api", "rest", "graphql", "ci/cd", "pipeline", "terraform",
|
||||
"aws", "azure", "cloud", "microservice", "frontend", "backend",
|
||||
"database", "sql", "mongodb", "redis",
|
||||
}
|
||||
|
||||
bahn_keywords = {
|
||||
"infrago", "db", "confluence", "jira", "openshift",
|
||||
"pipeship", "cnp", "einfachbahn",
|
||||
}
|
||||
|
||||
for kw in tech_keywords:
|
||||
if kw in content_lower:
|
||||
tags.append(kw)
|
||||
|
||||
for kw in bahn_keywords:
|
||||
if kw in content_lower:
|
||||
tags.append(kw)
|
||||
|
||||
# Limit tags to reasonable number
|
||||
return tags[:10]
|
||||
|
||||
|
||||
def generate_wiki_links(people: list[str], projects: list[str]) -> dict[str, list[str]]:
|
||||
"""Generate wiki-links for detected entities.
|
||||
|
||||
Args:
|
||||
people: List of detected person names.
|
||||
projects: List of detected project names.
|
||||
|
||||
Returns:
|
||||
Dict with 'people' and 'projects' wiki-link lists.
|
||||
"""
|
||||
people_links = [f"[[people/{_slugify(name)}]]" for name in people]
|
||||
project_links = [f"[[projects/{_slugify(proj)}]]" for proj in projects]
|
||||
return {"people": people_links, "projects": project_links}
|
||||
|
||||
|
||||
def generate_action_items_section(action_items: list[ActionItem]) -> str:
|
||||
"""Generate a markdown section for action items.
|
||||
|
||||
Args:
|
||||
action_items: List of ActionItem instances.
|
||||
|
||||
Returns:
|
||||
Markdown string with ## Action Items heading and list.
|
||||
"""
|
||||
if not action_items:
|
||||
return ""
|
||||
|
||||
lines = ["## Action Items", ""]
|
||||
for item in action_items:
|
||||
parts = [f"- [ ] {item.description}"]
|
||||
annotations: list[str] = []
|
||||
if item.assignee:
|
||||
annotations.append(f"@{item.assignee}")
|
||||
if item.deadline:
|
||||
annotations.append(f"bis {item.deadline}")
|
||||
if annotations:
|
||||
parts[0] += f" ({', '.join(annotations)})"
|
||||
lines.append(parts[0])
|
||||
|
||||
lines.append("") # trailing newline
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EnrichmentAgent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EnrichmentAgent:
|
||||
"""Content analysis and enrichment agent.
|
||||
|
||||
Provider-Strategie (Priorität):
|
||||
1. Kiro AI-Agent (primär) – nutzt den laufenden Orchestrator-Agenten direkt,
|
||||
keine externe Abhängigkeit, kein API-Key nötig
|
||||
2. Regelbasierte Heuristiken als Fallback wenn kein LLM verfügbar
|
||||
|
||||
Im aktuellen MVP werden nur die regelbasierten Heuristiken verwendet.
|
||||
Die LLM-Integration (Kiro/LiteLLM) wird als Stub vorgehalten.
|
||||
"""
|
||||
|
||||
def __init__(self, provider: str = "kiro", model: str = "") -> None:
|
||||
"""Initialize with LLM config.
|
||||
|
||||
Args:
|
||||
provider: LLM provider ('kiro' or 'litellm'). Defaults to 'kiro'.
|
||||
model: Model identifier (empty = provider default).
|
||||
"""
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
self._provider_available = self._check_provider_available()
|
||||
|
||||
def _check_provider_available(self) -> bool:
|
||||
"""Check if the configured LLM provider is available.
|
||||
|
||||
For now, rule-based heuristics are always available.
|
||||
The Kiro/LiteLLM check is a stub for future implementation.
|
||||
|
||||
Returns:
|
||||
True if provider is available, False otherwise.
|
||||
"""
|
||||
# Rule-based heuristics are always available
|
||||
# When LLM providers are implemented, this will actually check connectivity
|
||||
if self.provider == "kiro":
|
||||
# Kiro provider: in MVP we fall back to heuristics which are always available
|
||||
return True
|
||||
elif self.provider == "litellm":
|
||||
# LiteLLM would require env vars LLM_PROVIDER, LLM_MODEL
|
||||
# Stub: not available in MVP
|
||||
return False
|
||||
return False
|
||||
|
||||
def enrich(self, content: str, source_type: str = "") -> EnrichmentResult:
|
||||
"""Analyze content and generate metadata.
|
||||
|
||||
Uses rule-based heuristics to extract:
|
||||
- Title from first heading or first line
|
||||
- Category from keyword matching
|
||||
- Tags from domain keywords
|
||||
- People from name patterns and @mentions
|
||||
- Projects from explicit references
|
||||
- Action items from TODO/checkbox patterns
|
||||
|
||||
Only entities with confidence >= 0.7 are included in the output.
|
||||
|
||||
Args:
|
||||
content: The text content to analyze.
|
||||
source_type: Optional hint about the source type.
|
||||
|
||||
Returns:
|
||||
EnrichmentResult with extracted metadata.
|
||||
"""
|
||||
if not content or not content.strip():
|
||||
return EnrichmentResult(
|
||||
title="Untitled",
|
||||
category="inbox",
|
||||
confidence=0.0,
|
||||
enrichment_pending=not self._provider_available,
|
||||
)
|
||||
|
||||
# If provider is not available, return minimal result with pending flag
|
||||
if not self._provider_available:
|
||||
title = _extract_title(content)
|
||||
return EnrichmentResult(
|
||||
title=title,
|
||||
category="inbox",
|
||||
confidence=0.0,
|
||||
enrichment_pending=True,
|
||||
)
|
||||
|
||||
# Extract title
|
||||
title = _extract_title(content)
|
||||
|
||||
# Detect category
|
||||
category, cat_confidence = _detect_category(content, source_type)
|
||||
|
||||
# Extract tags
|
||||
tags = _extract_tags(content, category)
|
||||
|
||||
# Extract people (apply confidence threshold)
|
||||
raw_people = _extract_people(content)
|
||||
people = [name for name, conf in raw_people if conf >= CONFIDENCE_THRESHOLD]
|
||||
|
||||
# Extract projects (apply confidence threshold)
|
||||
raw_projects = _extract_projects(content)
|
||||
projects = [name for name, conf in raw_projects if conf >= CONFIDENCE_THRESHOLD]
|
||||
|
||||
# Extract action items
|
||||
action_items = _extract_action_items(content)
|
||||
|
||||
# Compute overall confidence
|
||||
# Based on: how much structure was found in the content
|
||||
signals = [
|
||||
cat_confidence,
|
||||
min(1.0, len(tags) * 0.2) if tags else 0.0,
|
||||
0.8 if people else 0.0,
|
||||
0.8 if action_items else 0.0,
|
||||
]
|
||||
overall_confidence = sum(signals) / len(signals) if signals else 0.3
|
||||
|
||||
return EnrichmentResult(
|
||||
title=title,
|
||||
category=category,
|
||||
tags=tags,
|
||||
people=people,
|
||||
projects=projects,
|
||||
action_items=action_items,
|
||||
confidence=round(overall_confidence, 2),
|
||||
enrichment_pending=False,
|
||||
)
|
||||
|
||||
def classify_relevance(self, content: str) -> float:
|
||||
"""Score content relevance (0.0 = irrelevant, 1.0 = highly relevant).
|
||||
|
||||
Simple heuristic based on:
|
||||
- Content length (very short = likely low relevance)
|
||||
- Structural elements (headings, lists, links)
|
||||
- Domain keywords presence
|
||||
- Action items presence
|
||||
|
||||
Args:
|
||||
content: The text content to score.
|
||||
|
||||
Returns:
|
||||
Float between 0.0 and 1.0 indicating relevance.
|
||||
"""
|
||||
if not content or not content.strip():
|
||||
return 0.0
|
||||
|
||||
score = 0.0
|
||||
content_stripped = content.strip()
|
||||
|
||||
# Length-based scoring
|
||||
length = len(content_stripped)
|
||||
if length < 20:
|
||||
score += 0.1
|
||||
elif length < 100:
|
||||
score += 0.3
|
||||
elif length < 500:
|
||||
score += 0.5
|
||||
elif length < 2000:
|
||||
score += 0.7
|
||||
else:
|
||||
score += 0.8
|
||||
|
||||
# Structural elements bonus
|
||||
has_headings = bool(re.search(r"^#+\s", content, re.MULTILINE))
|
||||
has_lists = bool(re.search(r"^[-*]\s", content, re.MULTILINE))
|
||||
has_links = bool(re.search(r"\[.+?\]\(.+?\)", content))
|
||||
|
||||
if has_headings:
|
||||
score += 0.1
|
||||
if has_lists:
|
||||
score += 0.05
|
||||
if has_links:
|
||||
score += 0.05
|
||||
|
||||
# Action items bonus
|
||||
for pattern in ACTION_ITEM_PATTERNS:
|
||||
if pattern.search(content):
|
||||
score += 0.1
|
||||
break
|
||||
|
||||
# Cap at 1.0
|
||||
return min(1.0, round(score, 2))
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Git-Integration – Automatisches Committen von Wissensänderungen.
|
||||
|
||||
Erstellt strukturierte Git-Commits nach erfolgreicher Ingestion
|
||||
mit konventionellen Commit-Messages.
|
||||
|
||||
Commit-Message-Format:
|
||||
knowledge({context}): {action} {count} artifacts from {source}
|
||||
knowledge({context}): capture "{title}"
|
||||
|
||||
Verwendet subprocess für Git-Aufrufe (kein gitpython).
|
||||
Fehler werden graceful behandelt: Warnung loggen, Dateien beibehalten.
|
||||
|
||||
Requirements: 16.1, 16.2, 16.3, 16.4, 16.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_commit_message(
|
||||
context: str,
|
||||
action: str,
|
||||
count: int,
|
||||
source: str,
|
||||
) -> str:
|
||||
"""Generiert eine konventionelle Commit-Message für Knowledge-Änderungen.
|
||||
|
||||
Format: knowledge({context}): {action} {count} artifacts from {source}
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (bahn, dhive, privat).
|
||||
action: Die durchgeführte Aktion (ingest, capture, update, etc.).
|
||||
count: Anzahl der verarbeiteten Artefakte.
|
||||
source: Die Quelle der Artefakte (confluence, jira, inbox, etc.).
|
||||
|
||||
Returns:
|
||||
Die formatierte Commit-Message.
|
||||
"""
|
||||
return f"knowledge({context}): {action} {count} artifacts from {source}"
|
||||
|
||||
|
||||
def generate_capture_commit_message(context: str, title: str) -> str:
|
||||
"""Generiert eine Commit-Message für Quick-Capture-Artefakte.
|
||||
|
||||
Format: knowledge({context}): capture "{title}"
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (bahn, dhive, privat).
|
||||
title: Der Titel des erfassten Artefakts.
|
||||
|
||||
Returns:
|
||||
Die formatierte Commit-Message.
|
||||
"""
|
||||
return f'knowledge({context}): capture "{title}"'
|
||||
|
||||
|
||||
def auto_commit(
|
||||
context: str,
|
||||
action: str,
|
||||
count: int,
|
||||
source: str,
|
||||
paths: list[Path],
|
||||
*,
|
||||
no_commit: bool = False,
|
||||
auto_commit_enabled: bool = True,
|
||||
cwd: Path | None = None,
|
||||
) -> str:
|
||||
"""Staged und committed Änderungen an den angegebenen Pfaden.
|
||||
|
||||
Respektiert --no-commit-Flag und git.auto_commit-Config.
|
||||
Behandelt Git-Fehler graceful (Warnung, Dateien bleiben erhalten).
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (bahn, dhive, privat).
|
||||
action: Die durchgeführte Aktion (ingest, capture, etc.).
|
||||
count: Anzahl der verarbeiteten Artefakte.
|
||||
source: Die Quelle der Artefakte.
|
||||
paths: Liste der zu committenden Dateipfade.
|
||||
no_commit: Wenn True, wird kein Commit erzeugt (--no-commit Flag).
|
||||
auto_commit_enabled: Wert aus git.auto_commit in sources.yaml.
|
||||
cwd: Arbeitsverzeichnis für Git-Befehle. Wenn None, wird das
|
||||
übergeordnete Verzeichnis des ersten Pfades verwendet.
|
||||
|
||||
Returns:
|
||||
Den Commit-SHA bei Erfolg, oder einen leeren String wenn kein
|
||||
Commit erzeugt wurde (no_commit, disabled, Fehler, keine Pfade).
|
||||
"""
|
||||
# Keine Pfade → nichts zu committen
|
||||
if not paths:
|
||||
logger.debug("auto_commit: Keine Pfade angegeben, überspringe Commit.")
|
||||
return ""
|
||||
|
||||
# --no-commit Flag hat höchste Priorität
|
||||
if no_commit:
|
||||
logger.info("auto_commit: --no-commit gesetzt, überspringe Git-Commit.")
|
||||
return ""
|
||||
|
||||
# git.auto_commit aus Config prüfen
|
||||
if not auto_commit_enabled:
|
||||
logger.info(
|
||||
"auto_commit: git.auto_commit ist deaktiviert, überspringe Git-Commit."
|
||||
)
|
||||
return ""
|
||||
|
||||
# Arbeitsverzeichnis bestimmen
|
||||
if cwd is None:
|
||||
cwd = paths[0].parent
|
||||
|
||||
# Commit-Message generieren
|
||||
message = generate_commit_message(context, action, count, source)
|
||||
|
||||
return _execute_git_commit(paths, message, cwd)
|
||||
|
||||
|
||||
def auto_commit_capture(
|
||||
context: str,
|
||||
title: str,
|
||||
paths: list[Path],
|
||||
*,
|
||||
no_commit: bool = False,
|
||||
auto_commit_enabled: bool = True,
|
||||
cwd: Path | None = None,
|
||||
) -> str:
|
||||
"""Staged und committed ein Quick-Capture-Artefakt.
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (bahn, dhive, privat).
|
||||
title: Der Titel des erfassten Artefakts.
|
||||
paths: Liste der zu committenden Dateipfade.
|
||||
no_commit: Wenn True, wird kein Commit erzeugt.
|
||||
auto_commit_enabled: Wert aus git.auto_commit in sources.yaml.
|
||||
cwd: Arbeitsverzeichnis für Git-Befehle.
|
||||
|
||||
Returns:
|
||||
Den Commit-SHA bei Erfolg, oder einen leeren String.
|
||||
"""
|
||||
if not paths:
|
||||
return ""
|
||||
|
||||
if no_commit:
|
||||
logger.info("auto_commit_capture: --no-commit gesetzt, überspringe.")
|
||||
return ""
|
||||
|
||||
if not auto_commit_enabled:
|
||||
logger.info("auto_commit_capture: git.auto_commit deaktiviert, überspringe.")
|
||||
return ""
|
||||
|
||||
if cwd is None:
|
||||
cwd = paths[0].parent
|
||||
|
||||
message = generate_capture_commit_message(context, title)
|
||||
|
||||
return _execute_git_commit(paths, message, cwd)
|
||||
|
||||
|
||||
def _execute_git_commit(
|
||||
paths: list[Path],
|
||||
message: str,
|
||||
cwd: Path,
|
||||
) -> str:
|
||||
"""Führt git add + git commit aus.
|
||||
|
||||
Bei Fehlern wird eine Warnung geloggt und ein leerer String zurückgegeben.
|
||||
Die erstellten Dateien bleiben in jedem Fall erhalten.
|
||||
|
||||
Args:
|
||||
paths: Zu stagenden Dateipfade.
|
||||
message: Die Commit-Message.
|
||||
cwd: Arbeitsverzeichnis für Git-Befehle.
|
||||
|
||||
Returns:
|
||||
Den Commit-SHA bei Erfolg, oder einen leeren String bei Fehler.
|
||||
"""
|
||||
try:
|
||||
# Stage files
|
||||
str_paths = [str(p) for p in paths]
|
||||
_run_git(["add", "--"] + str_paths, cwd=cwd)
|
||||
|
||||
# Commit
|
||||
_run_git(["commit", "-m", message], cwd=cwd)
|
||||
|
||||
# SHA des letzten Commits holen
|
||||
result = _run_git(["rev-parse", "HEAD"], cwd=cwd)
|
||||
sha = result.stdout.strip()
|
||||
logger.info(f"Git-Commit erstellt: {sha[:8]} – {message}")
|
||||
return sha
|
||||
|
||||
except GitCommandError as e:
|
||||
logger.warning(
|
||||
f"Git-Commit fehlgeschlagen: {e}. "
|
||||
"Dateien wurden erstellt, aber nicht committed."
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def _run_git(
|
||||
args: list[str],
|
||||
cwd: Path,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
"""Führt einen Git-Befehl als Subprocess aus.
|
||||
|
||||
Args:
|
||||
args: Git-Subcommand und Argumente (ohne 'git' Präfix).
|
||||
cwd: Arbeitsverzeichnis.
|
||||
|
||||
Returns:
|
||||
Das CompletedProcess-Objekt bei Erfolg.
|
||||
|
||||
Raises:
|
||||
GitCommandError: Wenn der Git-Befehl fehlschlägt.
|
||||
"""
|
||||
cmd = ["git"] + args
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(cwd),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise GitCommandError(
|
||||
"Git ist nicht installiert oder nicht im PATH verfügbar."
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise GitCommandError(f"Git-Befehl hat Timeout überschritten: {cmd}")
|
||||
except OSError as e:
|
||||
raise GitCommandError(f"OS-Fehler beim Ausführen von Git: {e}")
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = result.stderr.strip()
|
||||
raise GitCommandError(
|
||||
f"Git-Befehl fehlgeschlagen (exit {result.returncode}): "
|
||||
f"{' '.join(cmd)}\n{stderr}"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class GitCommandError(Exception):
|
||||
"""Wird bei fehlgeschlagenen Git-Befehlen ausgelöst.
|
||||
|
||||
Diese Exception wird intern in _execute_git_commit abgefangen
|
||||
und in eine Warnung umgewandelt.
|
||||
"""
|
||||
|
||||
pass
|
||||
@@ -0,0 +1,715 @@
|
||||
"""IngestionPipeline – Orchestrator für die Wissensaufnahme.
|
||||
|
||||
Koordiniert den gesamten Ingestion-Workflow: Quellen laden, extrahieren,
|
||||
anreichern, speichern und indexieren.
|
||||
|
||||
Requirements: 2.2, 2.3, 2.4, 2.5, 2.6, 3.4, 3.5, 3.6, 12.3
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
Artifact,
|
||||
ArtifactMetadata,
|
||||
compute_content_hash,
|
||||
write_artifact,
|
||||
)
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.ingestion.config import (
|
||||
PipelineSettings,
|
||||
SourceConfig,
|
||||
load_sources_config,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.enrichment import (
|
||||
ActionItem,
|
||||
EnrichmentAgent,
|
||||
EnrichmentResult,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.git_integration import auto_commit
|
||||
from monorepo.knowledge.ingestion.router import ContextRouter, category_to_folder
|
||||
from monorepo.knowledge.integrations.orgmylife import OrgMyLifeIntegration
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestError:
|
||||
"""Structured error from ingestion pipeline."""
|
||||
|
||||
source: str
|
||||
error_type: str
|
||||
message: str
|
||||
timestamp: str = ""
|
||||
retry: bool = False
|
||||
artifact_path: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.timestamp:
|
||||
self.timestamp = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestionResult:
|
||||
"""Result of a full ingestion run."""
|
||||
|
||||
context: str
|
||||
processed: int = 0
|
||||
updated: int = 0
|
||||
skipped: int = 0
|
||||
errors: list[IngestError] = field(default_factory=list)
|
||||
tasks_created: int = 0
|
||||
commit_sha: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Maps source type names to SourceStrategy classes.
|
||||
# Populated by register_source_strategy() or by the pipeline at startup.
|
||||
_SOURCE_REGISTRY: dict[str, type[SourceStrategy]] = {}
|
||||
|
||||
|
||||
def register_source_strategy(source_type: str, strategy_class: type[SourceStrategy]) -> None:
|
||||
"""Register a source strategy class for a given source type.
|
||||
|
||||
Args:
|
||||
source_type: The source type name (e.g., 'markdown', 'confluence').
|
||||
strategy_class: The SourceStrategy subclass to use for this type.
|
||||
"""
|
||||
_SOURCE_REGISTRY[source_type] = strategy_class
|
||||
|
||||
|
||||
def get_source_strategy(source_type: str) -> SourceStrategy | None:
|
||||
"""Get an instantiated source strategy for the given type.
|
||||
|
||||
Args:
|
||||
source_type: The source type name.
|
||||
|
||||
Returns:
|
||||
An instance of the registered strategy, or None if not registered.
|
||||
"""
|
||||
strategy_class = _SOURCE_REGISTRY.get(source_type)
|
||||
if strategy_class is None:
|
||||
return None
|
||||
return strategy_class()
|
||||
|
||||
|
||||
def register_default_strategies() -> None:
|
||||
"""Register all built-in source strategies.
|
||||
|
||||
This is called automatically when an IngestionPipeline is created,
|
||||
ensuring all standard source types are available.
|
||||
"""
|
||||
from monorepo.knowledge.sources.chat import ChatSource
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource
|
||||
from monorepo.knowledge.sources.email import EmailSource
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.knowledge.sources.pdf import PDFSource
|
||||
from monorepo.knowledge.sources.wissensdatenbank import WissensdatenbankSource
|
||||
|
||||
defaults: dict[str, type[SourceStrategy]] = {
|
||||
"markdown": MarkdownSource,
|
||||
"pdf": PDFSource,
|
||||
"confluence": ConfluenceSource,
|
||||
"jira": JiraSource,
|
||||
"email": EmailSource,
|
||||
"chat": ChatSource,
|
||||
"wissensdatenbank": WissensdatenbankSource,
|
||||
}
|
||||
|
||||
for source_type, strategy_class in defaults.items():
|
||||
if source_type not in _SOURCE_REGISTRY:
|
||||
_SOURCE_REGISTRY[source_type] = strategy_class
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IngestionPipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class IngestionPipeline:
|
||||
"""Orchestrates the full ingestion workflow.
|
||||
|
||||
Pipeline steps:
|
||||
1. Load sources.yaml configuration
|
||||
2. For each enabled source: extract artifacts via SourceStrategy
|
||||
3. Enrich artifacts (titles, categories, tags, entities)
|
||||
4. Store artifacts in the correct knowledge folder
|
||||
5. Update YAML index
|
||||
6. Optionally commit changes via git
|
||||
|
||||
Fail-forward: individual source failures are logged and skipped,
|
||||
remaining sources continue processing.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
monorepo_root: Path,
|
||||
context: str,
|
||||
dry_run: bool = False,
|
||||
no_commit: bool = False,
|
||||
enrichment_agent: EnrichmentAgent | None = None,
|
||||
source_strategies: dict[str, SourceStrategy] | None = None,
|
||||
) -> None:
|
||||
"""Initialize the ingestion pipeline.
|
||||
|
||||
Args:
|
||||
monorepo_root: Root path of the monorepo.
|
||||
context: Target context (bahn, dhive, privat).
|
||||
dry_run: If True, do not write files.
|
||||
no_commit: If True, skip git commit.
|
||||
enrichment_agent: Optional custom enrichment agent (for testing).
|
||||
source_strategies: Optional custom strategy mapping (for testing).
|
||||
"""
|
||||
self.monorepo_root = monorepo_root
|
||||
self.context = context
|
||||
self.dry_run = dry_run
|
||||
self.no_commit = no_commit
|
||||
self.router = ContextRouter(monorepo_root)
|
||||
self.enrichment_agent = enrichment_agent or EnrichmentAgent()
|
||||
self._source_strategies = source_strategies or {}
|
||||
|
||||
# Ensure all built-in source strategies are registered
|
||||
register_default_strategies()
|
||||
|
||||
@property
|
||||
def knowledge_path(self) -> Path:
|
||||
"""Path to the context's knowledge folder."""
|
||||
return self.monorepo_root / self.context / "knowledge"
|
||||
|
||||
@property
|
||||
def index_path(self) -> Path:
|
||||
"""Path to the context's YAML index."""
|
||||
return self.knowledge_path / "_index.yaml"
|
||||
|
||||
@property
|
||||
def sources_config_path(self) -> Path:
|
||||
"""Path to the context's sources.yaml."""
|
||||
return self.knowledge_path / "sources.yaml"
|
||||
|
||||
def _get_strategy(self, source_type: str) -> SourceStrategy | None:
|
||||
"""Get a source strategy, checking local overrides first.
|
||||
|
||||
Args:
|
||||
source_type: The source type name.
|
||||
|
||||
Returns:
|
||||
A SourceStrategy instance or None.
|
||||
"""
|
||||
if source_type in self._source_strategies:
|
||||
return self._source_strategies[source_type]
|
||||
return get_source_strategy(source_type)
|
||||
|
||||
def _load_index(self) -> YAMLIndex:
|
||||
"""Load the YAML index for this context."""
|
||||
index = YAMLIndex(self.index_path)
|
||||
index.load()
|
||||
return index
|
||||
|
||||
def _is_unchanged(self, artifact: Artifact, index: YAMLIndex) -> bool:
|
||||
"""Check if an artifact is unchanged based on content hash.
|
||||
|
||||
Args:
|
||||
artifact: The artifact to check.
|
||||
index: The current YAML index.
|
||||
|
||||
Returns:
|
||||
True if the artifact's content hash matches the index entry.
|
||||
"""
|
||||
content_hash = compute_content_hash(artifact.content)
|
||||
artifact_id = self._compute_artifact_id(artifact)
|
||||
existing = index.get_entry(artifact_id)
|
||||
if existing and existing.content_hash == content_hash:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _compute_artifact_id(self, artifact: Artifact) -> str:
|
||||
"""Compute a unique ID for an artifact.
|
||||
|
||||
Format: {context}/{type}/{filename_stem}
|
||||
|
||||
Args:
|
||||
artifact: The artifact.
|
||||
|
||||
Returns:
|
||||
String ID for the artifact.
|
||||
"""
|
||||
stem = artifact.file_path.stem if artifact.file_path.stem else "untitled"
|
||||
art_type = artifact.metadata.type or artifact.metadata.category or "inbox"
|
||||
return f"{self.context}/{art_type}/{stem}"
|
||||
|
||||
def _enrich_artifact(self, artifact: Artifact) -> Artifact:
|
||||
"""Enrich an artifact with metadata from the enrichment agent.
|
||||
|
||||
Args:
|
||||
artifact: The raw artifact.
|
||||
|
||||
Returns:
|
||||
The artifact with enriched metadata.
|
||||
"""
|
||||
result = self.enrichment_agent.enrich(
|
||||
artifact.content, artifact.metadata.source.get("type", "")
|
||||
)
|
||||
|
||||
# Apply enrichment results to metadata
|
||||
if not artifact.metadata.title or artifact.metadata.title == "Untitled":
|
||||
artifact.metadata.title = result.title
|
||||
|
||||
if not artifact.metadata.category:
|
||||
artifact.metadata.category = result.category
|
||||
|
||||
if not artifact.metadata.tags:
|
||||
artifact.metadata.tags = result.tags
|
||||
|
||||
if not artifact.metadata.people:
|
||||
artifact.metadata.people = [
|
||||
f"[[people/{_slugify_simple(p)}]]" for p in result.people
|
||||
]
|
||||
|
||||
if not artifact.metadata.projects:
|
||||
artifact.metadata.projects = [
|
||||
f"[[projects/{_slugify_simple(p)}]]" for p in result.projects
|
||||
]
|
||||
|
||||
artifact.metadata.enrichment_pending = result.enrichment_pending
|
||||
|
||||
# Compute content hash
|
||||
artifact.metadata.content_hash = compute_content_hash(artifact.content)
|
||||
|
||||
# Set source context
|
||||
if not artifact.metadata.source_context:
|
||||
artifact.metadata.source_context = self.context
|
||||
|
||||
return artifact
|
||||
|
||||
def _store_artifact(self, artifact: Artifact) -> Path:
|
||||
"""Store an artifact in the knowledge folder.
|
||||
|
||||
Args:
|
||||
artifact: The enriched artifact.
|
||||
|
||||
Returns:
|
||||
The path where the artifact was stored.
|
||||
"""
|
||||
category = artifact.metadata.category or "inbox"
|
||||
folder = category_to_folder(category)
|
||||
target_dir = self.knowledge_path / folder
|
||||
|
||||
if not self.dry_run:
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Determine filename
|
||||
filename = artifact.file_path.name
|
||||
if not filename:
|
||||
filename = "untitled.md"
|
||||
|
||||
target_path = target_dir / filename
|
||||
|
||||
if not self.dry_run:
|
||||
# Write the artifact file
|
||||
from monorepo.knowledge.artifact import _generate_frontmatter
|
||||
|
||||
frontmatter = _generate_frontmatter(artifact.metadata)
|
||||
content = frontmatter + artifact.content
|
||||
target_path.write_text(content, encoding="utf-8")
|
||||
|
||||
artifact.file_path = target_path
|
||||
return target_path
|
||||
|
||||
def _update_index(self, artifact: Artifact, index: YAMLIndex) -> None:
|
||||
"""Update the YAML index with the artifact's metadata.
|
||||
|
||||
Args:
|
||||
artifact: The processed artifact.
|
||||
index: The YAML index to update.
|
||||
"""
|
||||
artifact_id = self._compute_artifact_id(artifact)
|
||||
|
||||
entry = IndexEntry(
|
||||
id=artifact_id,
|
||||
title=artifact.metadata.title,
|
||||
type=artifact.metadata.type or artifact.metadata.category or "inbox",
|
||||
tags=artifact.metadata.tags,
|
||||
scope=self.context,
|
||||
summary="",
|
||||
path=str(artifact.file_path),
|
||||
content_hash=artifact.metadata.content_hash,
|
||||
links=[link.target for link in artifact.metadata.links],
|
||||
)
|
||||
|
||||
index.update_entry(entry)
|
||||
|
||||
def run(self) -> IngestionResult:
|
||||
"""Execute full pipeline: load config → extract → enrich → store → index → commit → tasks.
|
||||
|
||||
Returns:
|
||||
IngestionResult with processing statistics.
|
||||
"""
|
||||
result = IngestionResult(context=self.context)
|
||||
|
||||
# Load index
|
||||
index = self._load_index()
|
||||
|
||||
# Load source configurations
|
||||
sources: list[SourceConfig] = []
|
||||
settings = PipelineSettings()
|
||||
|
||||
if self.sources_config_path.exists():
|
||||
try:
|
||||
sources, settings = load_sources_config(self.sources_config_path)
|
||||
except Exception as e:
|
||||
result.errors.append(IngestError(
|
||||
source="sources.yaml",
|
||||
error_type="config",
|
||||
message=str(e),
|
||||
))
|
||||
return result
|
||||
|
||||
# Track written paths for git commit
|
||||
written_paths: list[Path] = []
|
||||
|
||||
# Track action items for OrgMyLife task creation
|
||||
all_action_items: list[tuple[str, list[ActionItem], str]] = []
|
||||
|
||||
# Process each enabled source
|
||||
for source_config in sources:
|
||||
if not source_config.enabled:
|
||||
continue
|
||||
|
||||
strategy = self._get_strategy(source_config.type)
|
||||
if strategy is None:
|
||||
logger.warning(
|
||||
f"No strategy registered for source type '{source_config.type}'"
|
||||
)
|
||||
result.errors.append(IngestError(
|
||||
source=source_config.name,
|
||||
error_type="config",
|
||||
message=f"No strategy for type '{source_config.type}'",
|
||||
))
|
||||
continue
|
||||
|
||||
# Extract artifacts from source
|
||||
try:
|
||||
extraction = strategy.extract(source_config, self.context)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Source '{source_config.name}' failed: {e}"
|
||||
)
|
||||
result.errors.append(IngestError(
|
||||
source=source_config.name,
|
||||
error_type="extraction",
|
||||
message=str(e),
|
||||
retry=True,
|
||||
))
|
||||
continue
|
||||
|
||||
# Record extraction errors
|
||||
for src_error in extraction.errors:
|
||||
result.errors.append(IngestError(
|
||||
source=src_error.source_name,
|
||||
error_type=src_error.error_type,
|
||||
message=src_error.message,
|
||||
timestamp=src_error.timestamp,
|
||||
retry=src_error.retry,
|
||||
))
|
||||
|
||||
result.skipped += extraction.skipped
|
||||
|
||||
# Process each extracted artifact
|
||||
for artifact in extraction.artifacts:
|
||||
# Check if unchanged (idempotent ingestion)
|
||||
if self._is_unchanged(artifact, index):
|
||||
result.skipped += 1
|
||||
continue
|
||||
|
||||
# Enrich
|
||||
artifact = self._enrich_artifact(artifact)
|
||||
|
||||
# Collect action items for OrgMyLife task creation
|
||||
enrichment_result = self.enrichment_agent.enrich(
|
||||
artifact.content, artifact.metadata.source.get("type", "")
|
||||
)
|
||||
if enrichment_result.action_items:
|
||||
artifact_id = self._compute_artifact_id(artifact)
|
||||
all_action_items.append((
|
||||
artifact_id,
|
||||
enrichment_result.action_items,
|
||||
str(artifact.file_path),
|
||||
))
|
||||
|
||||
# Store
|
||||
if not self.dry_run:
|
||||
stored_path = self._store_artifact(artifact)
|
||||
written_paths.append(stored_path)
|
||||
|
||||
# Update index
|
||||
self._update_index(artifact, index)
|
||||
|
||||
result.processed += 1
|
||||
|
||||
# Save index
|
||||
if not self.dry_run:
|
||||
index.save()
|
||||
written_paths.append(self.index_path)
|
||||
|
||||
# Git auto-commit (Requirement 16.1)
|
||||
if not self.dry_run and written_paths:
|
||||
commit_sha = auto_commit(
|
||||
context=self.context,
|
||||
action="ingest",
|
||||
count=result.processed,
|
||||
source="pipeline",
|
||||
paths=written_paths,
|
||||
no_commit=self.no_commit,
|
||||
auto_commit_enabled=settings.git.auto_commit,
|
||||
cwd=self.monorepo_root,
|
||||
)
|
||||
result.commit_sha = commit_sha
|
||||
|
||||
# OrgMyLife task creation for action items (Requirement 10.1)
|
||||
if not self.dry_run and settings.create_tasks and all_action_items:
|
||||
try:
|
||||
orgmylife = OrgMyLifeIntegration(
|
||||
context_path=self.knowledge_path,
|
||||
enabled=settings.create_tasks,
|
||||
)
|
||||
for artifact_id, action_items, source_path in all_action_items:
|
||||
mappings = orgmylife.create_tasks(
|
||||
artifact_id=artifact_id,
|
||||
action_items=action_items,
|
||||
source_path=source_path,
|
||||
)
|
||||
result.tasks_created += len(mappings)
|
||||
except Exception as e:
|
||||
logger.warning(f"OrgMyLife task creation failed: {e}")
|
||||
result.errors.append(IngestError(
|
||||
source="orgmylife",
|
||||
error_type="integration",
|
||||
message=str(e),
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
def run_with_artifacts(
|
||||
self,
|
||||
artifacts: list[Artifact],
|
||||
source_configs: list[SourceConfig] | None = None,
|
||||
) -> IngestionResult:
|
||||
"""Run pipeline with pre-extracted artifacts (for testing/programmatic use).
|
||||
|
||||
Args:
|
||||
artifacts: Pre-extracted artifacts to process.
|
||||
source_configs: Optional source configs (not used for extraction).
|
||||
|
||||
Returns:
|
||||
IngestionResult with processing statistics.
|
||||
"""
|
||||
result = IngestionResult(context=self.context)
|
||||
index = self._load_index()
|
||||
|
||||
for artifact in artifacts:
|
||||
# Check if unchanged (idempotent ingestion)
|
||||
if self._is_unchanged(artifact, index):
|
||||
result.skipped += 1
|
||||
continue
|
||||
|
||||
# Enrich
|
||||
artifact = self._enrich_artifact(artifact)
|
||||
|
||||
# Store
|
||||
if not self.dry_run:
|
||||
self._store_artifact(artifact)
|
||||
|
||||
# Update index
|
||||
self._update_index(artifact, index)
|
||||
result.processed += 1
|
||||
|
||||
# Save index
|
||||
if not self.dry_run:
|
||||
index.save()
|
||||
|
||||
return result
|
||||
|
||||
def process_inbox(self) -> IngestionResult:
|
||||
"""Process all items in {context}/knowledge/inbox/.
|
||||
|
||||
Reads all markdown files from the inbox, enriches them,
|
||||
and moves them to the appropriate category folder.
|
||||
After processing, commits changes and creates tasks for action items.
|
||||
|
||||
Returns:
|
||||
IngestionResult with processing statistics.
|
||||
"""
|
||||
result = IngestionResult(context=self.context)
|
||||
inbox_path = self.knowledge_path / "inbox"
|
||||
|
||||
if not inbox_path.exists():
|
||||
return result
|
||||
|
||||
index = self._load_index()
|
||||
|
||||
# Load settings for git/task config
|
||||
settings = PipelineSettings()
|
||||
if self.sources_config_path.exists():
|
||||
try:
|
||||
_, settings = load_sources_config(self.sources_config_path)
|
||||
except Exception:
|
||||
pass # Use defaults if config load fails
|
||||
|
||||
# Track written paths for git commit
|
||||
written_paths: list[Path] = []
|
||||
|
||||
# Track action items for OrgMyLife
|
||||
all_action_items: list[tuple[str, list[ActionItem], str]] = []
|
||||
|
||||
for file_path in inbox_path.glob("*.md"):
|
||||
if file_path.name == "index.md":
|
||||
continue
|
||||
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
except Exception as e:
|
||||
result.errors.append(IngestError(
|
||||
source=str(file_path),
|
||||
error_type="read",
|
||||
message=str(e),
|
||||
))
|
||||
continue
|
||||
|
||||
# Create artifact from inbox file
|
||||
artifact = Artifact(
|
||||
metadata=ArtifactMetadata(
|
||||
type="inbox",
|
||||
title=file_path.stem,
|
||||
source_context=self.context,
|
||||
created=date.today(),
|
||||
),
|
||||
content=content,
|
||||
file_path=file_path,
|
||||
)
|
||||
|
||||
# Check if unchanged
|
||||
if self._is_unchanged(artifact, index):
|
||||
result.skipped += 1
|
||||
continue
|
||||
|
||||
# Enrich
|
||||
artifact = self._enrich_artifact(artifact)
|
||||
|
||||
# Collect action items for OrgMyLife
|
||||
enrichment_result = self.enrichment_agent.enrich(
|
||||
artifact.content, "inbox"
|
||||
)
|
||||
if enrichment_result.action_items:
|
||||
artifact_id = self._compute_artifact_id(artifact)
|
||||
all_action_items.append((
|
||||
artifact_id,
|
||||
enrichment_result.action_items,
|
||||
str(artifact.file_path),
|
||||
))
|
||||
|
||||
# Store in categorized folder
|
||||
if not self.dry_run:
|
||||
stored_path = self._store_artifact(artifact)
|
||||
written_paths.append(stored_path)
|
||||
|
||||
# Update index
|
||||
self._update_index(artifact, index)
|
||||
result.processed += 1
|
||||
|
||||
# Save index
|
||||
if not self.dry_run:
|
||||
index.save()
|
||||
if result.processed > 0:
|
||||
written_paths.append(self.index_path)
|
||||
|
||||
# Git auto-commit
|
||||
if not self.dry_run and written_paths:
|
||||
commit_sha = auto_commit(
|
||||
context=self.context,
|
||||
action="categorize",
|
||||
count=result.processed,
|
||||
source="inbox",
|
||||
paths=written_paths,
|
||||
no_commit=self.no_commit,
|
||||
auto_commit_enabled=settings.git.auto_commit,
|
||||
cwd=self.monorepo_root,
|
||||
)
|
||||
result.commit_sha = commit_sha
|
||||
|
||||
# OrgMyLife task creation for action items
|
||||
if not self.dry_run and settings.create_tasks and all_action_items:
|
||||
try:
|
||||
orgmylife = OrgMyLifeIntegration(
|
||||
context_path=self.knowledge_path,
|
||||
enabled=settings.create_tasks,
|
||||
)
|
||||
for artifact_id, action_items, source_path in all_action_items:
|
||||
mappings = orgmylife.create_tasks(
|
||||
artifact_id=artifact_id,
|
||||
action_items=action_items,
|
||||
source_path=source_path,
|
||||
)
|
||||
result.tasks_created += len(mappings)
|
||||
except Exception as e:
|
||||
logger.warning(f"OrgMyLife task creation failed: {e}")
|
||||
result.errors.append(IngestError(
|
||||
source="orgmylife",
|
||||
error_type="integration",
|
||||
message=str(e),
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slugify_simple(name: str) -> str:
|
||||
"""Simple slugification for wiki-links.
|
||||
|
||||
Args:
|
||||
name: The name to slugify.
|
||||
|
||||
Returns:
|
||||
Lowercase, hyphen-separated string.
|
||||
"""
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
text = name.strip().lower()
|
||||
# German umlaut replacements
|
||||
replacements = {"ä": "ae", "ö": "oe", "ü": "ue", "ß": "ss"}
|
||||
for char, repl in replacements.items():
|
||||
text = text.replace(char, repl)
|
||||
# Remove diacritics
|
||||
text = unicodedata.normalize("NFD", text)
|
||||
text = "".join(c for c in text if unicodedata.category(c) != "Mn")
|
||||
# Replace non-alphanumeric with hyphens
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text)
|
||||
text = re.sub(r"-+", "-", text).strip("-")
|
||||
return text
|
||||
@@ -0,0 +1,180 @@
|
||||
"""ContextRouter – Routet Artefakte in den richtigen Wissenskontext.
|
||||
|
||||
Bestimmt anhand von Quellkonfiguration und Metadaten, in welchen
|
||||
Kontext (bahn, dhive, privat) ein Artefakt gehört und berechnet
|
||||
den Zielpfad im entsprechenden Knowledge-Folder.
|
||||
|
||||
Requirements: 1.4, 2.3, 3.2, 3.5, 9.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.ingestion.config import SourceConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_CONTEXTS = ("bahn", "dhive", "privat")
|
||||
|
||||
CATEGORY_FOLDER_MAP: dict[str, str] = {
|
||||
"meeting": "meetings",
|
||||
"decision": "decisions",
|
||||
"project": "projects",
|
||||
"reference": "references",
|
||||
"link": "links",
|
||||
"inbox": "inbox",
|
||||
}
|
||||
|
||||
DEFAULT_FOLDER = "inbox"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Model
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class RoutingDecision:
|
||||
"""Result of context routing."""
|
||||
|
||||
context: str # bahn, dhive, privat
|
||||
target_folder: str # knowledge subfolder (meetings, decisions, etc.)
|
||||
artifact_path: Path # full path for the artifact
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def category_to_folder(category: str) -> str:
|
||||
"""Map a category name to the corresponding knowledge subfolder.
|
||||
|
||||
Known categories are mapped to their plural folder name.
|
||||
Unknown categories default to 'inbox'.
|
||||
|
||||
Args:
|
||||
category: The category string (e.g. 'meeting', 'decision').
|
||||
|
||||
Returns:
|
||||
The folder name (e.g. 'meetings', 'decisions', 'inbox').
|
||||
"""
|
||||
return CATEGORY_FOLDER_MAP.get(category.lower(), DEFAULT_FOLDER)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ContextRouter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ContextRouter:
|
||||
"""Routes incoming data to the correct context knowledge folder.
|
||||
|
||||
Uses source configuration or working directory to determine the
|
||||
target context, then computes the full artifact path within the
|
||||
monorepo structure.
|
||||
"""
|
||||
|
||||
def __init__(self, monorepo_root: Path) -> None:
|
||||
"""Initialize with the monorepo root path.
|
||||
|
||||
Args:
|
||||
monorepo_root: Absolute path to the monorepo root directory.
|
||||
"""
|
||||
self.monorepo_root = monorepo_root
|
||||
|
||||
def determine_context(
|
||||
self,
|
||||
source_config: SourceConfig | None = None,
|
||||
cwd: Path | None = None,
|
||||
) -> str:
|
||||
"""Determine target context from source config or working directory.
|
||||
|
||||
Resolution order:
|
||||
1. If source_config is provided and has a 'context' param, use that.
|
||||
2. Otherwise derive from cwd by checking if path contains a known
|
||||
context segment (/bahn/, /dhive/, /privat/).
|
||||
3. If neither works, raise ValueError.
|
||||
|
||||
Args:
|
||||
source_config: Optional source configuration with context param.
|
||||
cwd: Optional working directory to derive context from.
|
||||
|
||||
Returns:
|
||||
The context string ('bahn', 'dhive', or 'privat').
|
||||
|
||||
Raises:
|
||||
ValueError: If context cannot be determined from either source.
|
||||
"""
|
||||
# Strategy 1: Extract from source_config params
|
||||
if source_config is not None:
|
||||
context_from_config = source_config.params.get("context", "")
|
||||
if context_from_config and context_from_config in VALID_CONTEXTS:
|
||||
return context_from_config
|
||||
|
||||
# Strategy 2: Derive from working directory path
|
||||
if cwd is not None:
|
||||
# Normalize to forward slashes for consistent matching
|
||||
path_str = str(cwd).replace("\\", "/")
|
||||
for ctx in VALID_CONTEXTS:
|
||||
if f"/{ctx}/" in path_str or path_str.endswith(f"/{ctx}"):
|
||||
return ctx
|
||||
|
||||
raise ValueError(
|
||||
"Kontext konnte nicht bestimmt werden. "
|
||||
"Weder source_config.params['context'] noch cwd enthalten "
|
||||
f"einen gültigen Kontext ({', '.join(VALID_CONTEXTS)})."
|
||||
)
|
||||
|
||||
def resolve_target_path(
|
||||
self, context: str, category: str, filename: str
|
||||
) -> Path:
|
||||
"""Compute full path: {root}/{context}/knowledge/{category_folder}/{filename}.
|
||||
|
||||
Args:
|
||||
context: The target context ('bahn', 'dhive', or 'privat').
|
||||
category: The artifact category (e.g. 'meeting', 'decision').
|
||||
filename: The filename for the artifact.
|
||||
|
||||
Returns:
|
||||
The full Path for the artifact within the knowledge folder.
|
||||
"""
|
||||
folder = category_to_folder(category)
|
||||
return self.monorepo_root / context / "knowledge" / folder / filename
|
||||
|
||||
def route(
|
||||
self,
|
||||
category: str,
|
||||
filename: str,
|
||||
source_config: SourceConfig | None = None,
|
||||
cwd: Path | None = None,
|
||||
) -> RoutingDecision:
|
||||
"""Full routing: determine context and compute target path.
|
||||
|
||||
Convenience method combining determine_context and resolve_target_path.
|
||||
|
||||
Args:
|
||||
category: The artifact category.
|
||||
filename: The artifact filename.
|
||||
source_config: Optional source configuration.
|
||||
cwd: Optional working directory.
|
||||
|
||||
Returns:
|
||||
A RoutingDecision with context, target_folder, and artifact_path.
|
||||
|
||||
Raises:
|
||||
ValueError: If context cannot be determined.
|
||||
"""
|
||||
context = self.determine_context(source_config, cwd)
|
||||
folder = category_to_folder(category)
|
||||
artifact_path = self.resolve_target_path(context, category, filename)
|
||||
return RoutingDecision(
|
||||
context=context,
|
||||
target_folder=folder,
|
||||
artifact_path=artifact_path,
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Externe Integrationen für das Knowledge-Management.
|
||||
|
||||
Bindet externe Systeme wie OrgMyLife, Kiro und LiteLLM an die
|
||||
Ingestion-Pipeline an. Enthält außerdem den Federation-Sync-Filter
|
||||
für die Knowledge-Ordner-Behandlung bei der Team-Repo-Synchronisation.
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.integrations.federation import (
|
||||
FederationSyncFilter,
|
||||
FederationSyncFilterResult,
|
||||
SyncExclusion,
|
||||
)
|
||||
from monorepo.knowledge.integrations.kiro_client import (
|
||||
KiroClient,
|
||||
KiroEnrichmentResult,
|
||||
KiroUnavailableError,
|
||||
check_kiro_available,
|
||||
create_kiro_client,
|
||||
)
|
||||
from monorepo.knowledge.integrations.orgmylife import (
|
||||
OrgMyLifeIntegration,
|
||||
TaskMapping,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FederationSyncFilter",
|
||||
"FederationSyncFilterResult",
|
||||
"KiroClient",
|
||||
"KiroEnrichmentResult",
|
||||
"KiroUnavailableError",
|
||||
"OrgMyLifeIntegration",
|
||||
"SyncExclusion",
|
||||
"TaskMapping",
|
||||
"check_kiro_available",
|
||||
"create_kiro_client",
|
||||
]
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Federation-Sync-Filter für Knowledge-Artefakte.
|
||||
|
||||
Erweitert die bestehende Federation-Logik um Knowledge-Ordner-Behandlung:
|
||||
- Exkludiert Artefakte mit `shareable: false` aus der Sync-Liste
|
||||
- Exkludiert ctx-guard-markierte sensible Artefakte
|
||||
- Stellt sicher, dass keine kontextübergreifenden Links oder Referenzen
|
||||
auf andere Kontexte in das Team-Repo gelangen
|
||||
|
||||
Requirements: 14.1, 14.2, 14.3, 14.4, 14.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import parse_frontmatter, ArtifactMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Valid monorepo contexts
|
||||
VALID_CONTEXTS = {"bahn", "dhive", "privat", "shared"}
|
||||
|
||||
# Pattern for cross-context links in frontmatter/content
|
||||
# Matches references like "bahn/knowledge/...", "dhive/...", "privat/..."
|
||||
_CROSS_CONTEXT_LINK_PATTERN = re.compile(
|
||||
r"(?:^|\s|/|\"|\')(?:bahn|dhive|privat|shared)/",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# Pattern for wiki-links referencing other contexts: [[bahn/...]], [[dhive/...]]
|
||||
_WIKI_LINK_CROSS_CONTEXT = re.compile(
|
||||
r"\[\[(?:bahn|dhive|privat|shared)/[^\]]+\]\]",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncExclusion:
|
||||
"""Grund für den Ausschluss eines Artefakts von der Synchronisation."""
|
||||
|
||||
path: Path
|
||||
reason: str # "not_shareable", "ctx_guard_sensitive", "cross_context_links"
|
||||
details: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FederationSyncFilterResult:
|
||||
"""Ergebnis der Federation-Sync-Filterung."""
|
||||
|
||||
included: list[Path] = field(default_factory=list)
|
||||
excluded: list[SyncExclusion] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return len(self.included) + len(self.excluded)
|
||||
|
||||
|
||||
class FederationSyncFilter:
|
||||
"""Filtert Knowledge-Artefakte für die Federation-Synchronisation.
|
||||
|
||||
Stellt sicher, dass nur teilbare, nicht-sensible Artefakte ohne
|
||||
kontextübergreifende Referenzen in das Team-Repo synchronisiert werden.
|
||||
|
||||
Die Filterung respektiert die bestehende team-repos.yaml-Konfiguration
|
||||
und behandelt den Knowledge-Ordner als Teil des Kontextordners
|
||||
(keine separate Sync-Konfiguration nötig).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
monorepo_root: Path,
|
||||
context: str,
|
||||
ctx_guard: Any | None = None,
|
||||
) -> None:
|
||||
"""Initialisiert den Federation-Sync-Filter.
|
||||
|
||||
Args:
|
||||
monorepo_root: Wurzelverzeichnis des Monorepos.
|
||||
context: Arbeitskontext (bahn, dhive, privat).
|
||||
ctx_guard: Optionaler ContextGuard für Sensitivitätsprüfung.
|
||||
"""
|
||||
self.monorepo_root = monorepo_root
|
||||
self.context = context
|
||||
self.ctx_guard = ctx_guard
|
||||
self._other_contexts = VALID_CONTEXTS - {context}
|
||||
|
||||
def filter_knowledge_artifacts(
|
||||
self, knowledge_path: Path | None = None
|
||||
) -> FederationSyncFilterResult:
|
||||
"""Filtert alle Artefakte im Knowledge-Ordner für die Sync.
|
||||
|
||||
Scannt den Knowledge-Ordner des Kontexts und bestimmt für jedes
|
||||
Artefakt, ob es synchronisiert werden darf.
|
||||
|
||||
Args:
|
||||
knowledge_path: Optionaler Pfad zum Knowledge-Ordner.
|
||||
Standardmäßig: {monorepo_root}/{context}/knowledge/
|
||||
|
||||
Returns:
|
||||
FederationSyncFilterResult mit included/excluded Listen.
|
||||
"""
|
||||
if knowledge_path is None:
|
||||
knowledge_path = self.monorepo_root / self.context / "knowledge"
|
||||
|
||||
result = FederationSyncFilterResult()
|
||||
|
||||
if not knowledge_path.exists():
|
||||
return result
|
||||
|
||||
# Scan all markdown files in the knowledge folder
|
||||
for md_file in knowledge_path.rglob("*.md"):
|
||||
exclusion = self._check_artifact(md_file)
|
||||
if exclusion is not None:
|
||||
result.excluded.append(exclusion)
|
||||
logger.info(
|
||||
"Federation-Sync: Artefakt ausgeschlossen: %s (%s)",
|
||||
md_file.name,
|
||||
exclusion.reason,
|
||||
)
|
||||
else:
|
||||
result.included.append(md_file)
|
||||
|
||||
return result
|
||||
|
||||
def get_sync_file_list(
|
||||
self, knowledge_path: Path | None = None
|
||||
) -> list[Path]:
|
||||
"""Gibt die Liste der Dateien zurück, die synchronisiert werden dürfen.
|
||||
|
||||
Convenience-Methode, die nur die included-Pfade zurückgibt.
|
||||
|
||||
Args:
|
||||
knowledge_path: Optionaler Pfad zum Knowledge-Ordner.
|
||||
|
||||
Returns:
|
||||
Liste der Pfade, die in das Team-Repo synchronisiert werden dürfen.
|
||||
"""
|
||||
result = self.filter_knowledge_artifacts(knowledge_path)
|
||||
return result.included
|
||||
|
||||
def should_sync(self, file_path: Path) -> bool:
|
||||
"""Prüft ob eine einzelne Datei synchronisiert werden darf.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Artefakt-Datei.
|
||||
|
||||
Returns:
|
||||
True wenn das Artefakt synchronisiert werden darf.
|
||||
"""
|
||||
return self._check_artifact(file_path) is None
|
||||
|
||||
def _check_artifact(self, file_path: Path) -> SyncExclusion | None:
|
||||
"""Prüft ein einzelnes Artefakt auf Sync-Ausschluss.
|
||||
|
||||
Prüft in folgender Reihenfolge:
|
||||
1. shareable: false → ausschließen
|
||||
2. ctx-guard-sensibel → ausschließen
|
||||
3. Kontextübergreifende Links → ausschließen
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Markdown-Datei.
|
||||
|
||||
Returns:
|
||||
SyncExclusion wenn das Artefakt ausgeschlossen wird, sonst None.
|
||||
"""
|
||||
# Nur Markdown-Dateien prüfen
|
||||
if file_path.suffix.lower() not in (".md", ".markdown"):
|
||||
return None
|
||||
|
||||
# Skip non-artifact files like index.md without frontmatter
|
||||
try:
|
||||
artifact = parse_frontmatter(file_path)
|
||||
except (ValueError, FileNotFoundError, OSError):
|
||||
# Kein gültiges Frontmatter → als normales Markdown durchlassen
|
||||
return None
|
||||
|
||||
metadata = artifact.metadata
|
||||
|
||||
# 1. Check shareable field (Req 14.4)
|
||||
if not metadata.shareable:
|
||||
return SyncExclusion(
|
||||
path=file_path,
|
||||
reason="not_shareable",
|
||||
details=f"Artefakt '{metadata.title}' hat shareable: false",
|
||||
)
|
||||
|
||||
# 2. Check ctx-guard sensitivity (Req 14.5)
|
||||
if self._is_ctx_guard_sensitive(file_path):
|
||||
return SyncExclusion(
|
||||
path=file_path,
|
||||
reason="ctx_guard_sensitive",
|
||||
details=f"Artefakt '{metadata.title}' ist ctx-guard-geschützt",
|
||||
)
|
||||
|
||||
# 3. Check for cross-context links (Req 14.3)
|
||||
cross_ctx_issue = self._has_cross_context_references(
|
||||
artifact.content, metadata
|
||||
)
|
||||
if cross_ctx_issue:
|
||||
return SyncExclusion(
|
||||
path=file_path,
|
||||
reason="cross_context_links",
|
||||
details=cross_ctx_issue,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _is_ctx_guard_sensitive(self, file_path: Path) -> bool:
|
||||
"""Prüft ob eine Datei über ctx-guard als sensibel markiert ist.
|
||||
|
||||
Nutzt den bestehenden ContextGuard-Mechanismus: Wenn ein
|
||||
externer Kontext (z.B. "shared") keinen Zugriff auf die Datei hat,
|
||||
gilt sie als sensibel und wird von der Sync ausgeschlossen.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
|
||||
Returns:
|
||||
True wenn die Datei sensibel ist.
|
||||
"""
|
||||
if self.ctx_guard is None:
|
||||
return False
|
||||
|
||||
# A file is considered sensitive if the "shared" context cannot access it.
|
||||
# This uses the existing ContextGuard mechanism to determine sensitivity.
|
||||
try:
|
||||
return not self.ctx_guard.check_access("shared", file_path)
|
||||
except (FileNotFoundError, ValueError):
|
||||
# If ctx-guard can't determine access, assume safe
|
||||
return False
|
||||
|
||||
def _has_cross_context_references(
|
||||
self, content: str, metadata: ArtifactMetadata
|
||||
) -> str:
|
||||
"""Prüft ob ein Artefakt kontextübergreifende Referenzen enthält.
|
||||
|
||||
Untersucht sowohl den Content als auch die Frontmatter-Links
|
||||
auf Verweise auf andere Kontexte.
|
||||
|
||||
Args:
|
||||
content: Markdown-Inhalt des Artefakts.
|
||||
metadata: Geparste Metadaten.
|
||||
|
||||
Returns:
|
||||
Beschreibung des Problems oder leerer String wenn keine
|
||||
kontextübergreifenden Referenzen gefunden wurden.
|
||||
"""
|
||||
issues: list[str] = []
|
||||
|
||||
# Check frontmatter links for cross-context references
|
||||
for link in metadata.links:
|
||||
target = link.target
|
||||
for other_ctx in self._other_contexts:
|
||||
if target.startswith(f"{other_ctx}/") or f"/{other_ctx}/" in target:
|
||||
issues.append(
|
||||
f"Link-Ziel verweist auf Kontext '{other_ctx}': {target}"
|
||||
)
|
||||
|
||||
# Check people references for cross-context paths
|
||||
for person_ref in metadata.people:
|
||||
for other_ctx in self._other_contexts:
|
||||
if f"{other_ctx}/" in person_ref:
|
||||
issues.append(
|
||||
f"People-Referenz verweist auf Kontext '{other_ctx}': {person_ref}"
|
||||
)
|
||||
|
||||
# Check project references for cross-context paths
|
||||
for project_ref in metadata.projects:
|
||||
for other_ctx in self._other_contexts:
|
||||
if f"{other_ctx}/" in project_ref:
|
||||
issues.append(
|
||||
f"Project-Referenz verweist auf Kontext '{other_ctx}': {project_ref}"
|
||||
)
|
||||
|
||||
# Check content for wiki-links to other contexts
|
||||
for match in _WIKI_LINK_CROSS_CONTEXT.finditer(content):
|
||||
link_text = match.group(0)
|
||||
# Only flag if it references a different context
|
||||
for other_ctx in self._other_contexts:
|
||||
if f"[[{other_ctx}/" in link_text:
|
||||
issues.append(
|
||||
f"Wiki-Link verweist auf Kontext '{other_ctx}': {link_text}"
|
||||
)
|
||||
|
||||
if issues:
|
||||
return "; ".join(issues)
|
||||
|
||||
return ""
|
||||
@@ -0,0 +1,316 @@
|
||||
"""Kiro AI-Agent Client – LLM-Provider für Enrichment und OCR.
|
||||
|
||||
Stellt das Interface für den Kiro AI-Agent als primären LLM-Provider bereit.
|
||||
Da Kiro der laufende Orchestrator-Agent selbst ist, arbeitet die reale
|
||||
Invokation über die Agent-Runtime (kein REST-Call). Diese Klasse definiert
|
||||
das Interface, das der EnrichmentAgent aufruft, und implementiert die
|
||||
Fallback-Logik bei Nicht-Verfügbarkeit.
|
||||
|
||||
Nutzung:
|
||||
client = create_kiro_client()
|
||||
if client is not None:
|
||||
result = client.enrich_content("Meeting notes ...", "confluence")
|
||||
text = client.ocr_image(Path("scan.png"))
|
||||
else:
|
||||
# Kiro nicht verfügbar → enrichment_pending: true setzen
|
||||
...
|
||||
|
||||
Requirements: 9.3, 9.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class KiroUnavailableError(Exception):
|
||||
"""Raised when the Kiro AI-Agent is not available.
|
||||
|
||||
This indicates that:
|
||||
- The Kiro agent session is not active
|
||||
- The caller should set `enrichment_pending: true` on the artifact
|
||||
- The artifact can be re-enriched on a subsequent run when Kiro is available
|
||||
"""
|
||||
|
||||
def __init__(self, reason: str = "Kiro AI-Agent session is not active") -> None:
|
||||
self.reason = reason
|
||||
super().__init__(reason)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class KiroEnrichmentResult:
|
||||
"""Result from Kiro-based content enrichment.
|
||||
|
||||
Attributes:
|
||||
title: Generated or extracted title.
|
||||
category: Content classification (meeting, decision, project, reference, link, inbox).
|
||||
tags: Generated tags.
|
||||
people: Detected person names.
|
||||
projects: Detected project references.
|
||||
action_items: Extracted action item descriptions.
|
||||
confidence: Overall confidence score (0.0 - 1.0).
|
||||
"""
|
||||
|
||||
title: str = ""
|
||||
category: str = "inbox"
|
||||
tags: list[str] = field(default_factory=list)
|
||||
people: list[str] = field(default_factory=list)
|
||||
projects: list[str] = field(default_factory=list)
|
||||
action_items: list[str] = field(default_factory=list)
|
||||
confidence: float = 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol (Interface Definition)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class KiroClientProtocol(Protocol):
|
||||
"""Protocol defining the Kiro client interface.
|
||||
|
||||
Any implementation (real agent runtime, mock, or alternative provider)
|
||||
must satisfy this protocol for the EnrichmentAgent to use it.
|
||||
"""
|
||||
|
||||
def enrich_content(self, content: str, source_type: str = "") -> KiroEnrichmentResult:
|
||||
"""Analyze content and return enrichment metadata.
|
||||
|
||||
Args:
|
||||
content: The text content to analyze.
|
||||
source_type: Optional hint about content origin (e.g., "confluence", "email").
|
||||
|
||||
Returns:
|
||||
KiroEnrichmentResult with extracted metadata.
|
||||
|
||||
Raises:
|
||||
KiroUnavailableError: If the Kiro agent is not reachable.
|
||||
"""
|
||||
...
|
||||
|
||||
def ocr_image(self, image_path: Path) -> str:
|
||||
"""Extract text from an image using Kiro Vision.
|
||||
|
||||
Args:
|
||||
image_path: Path to the image file (PNG, JPG, JPEG).
|
||||
|
||||
Returns:
|
||||
Extracted text content from the image.
|
||||
|
||||
Raises:
|
||||
KiroUnavailableError: If the Kiro agent is not reachable.
|
||||
FileNotFoundError: If the image file does not exist.
|
||||
"""
|
||||
...
|
||||
|
||||
@property
|
||||
def is_available(self) -> bool:
|
||||
"""Check if the Kiro agent is currently available.
|
||||
|
||||
Returns:
|
||||
True if the agent session is active and ready.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class KiroClient:
|
||||
"""Wrapper for the Kiro AI-Agent as LLM provider.
|
||||
|
||||
Since Kiro IS the running agent itself, the actual LLM invocations happen
|
||||
through the agent runtime (not via REST API). This class defines the
|
||||
interface and implements availability checking + fallback signaling.
|
||||
|
||||
In the current implementation, Kiro availability is determined by an
|
||||
environment marker or explicit flag. When Kiro is unavailable (e.g., in
|
||||
CI pipelines or batch processing without an active agent session), all
|
||||
methods raise KiroUnavailableError so the caller can set
|
||||
`enrichment_pending: true`.
|
||||
|
||||
Usage in the EnrichmentAgent:
|
||||
client = create_kiro_client()
|
||||
if client is not None:
|
||||
try:
|
||||
result = client.enrich_content(text)
|
||||
except KiroUnavailableError:
|
||||
# Set enrichment_pending: true
|
||||
pass
|
||||
"""
|
||||
|
||||
def __init__(self, *, force_available: bool | None = None) -> None:
|
||||
"""Initialize the Kiro client.
|
||||
|
||||
Args:
|
||||
force_available: Override availability detection.
|
||||
- None: auto-detect from environment (default)
|
||||
- True: force client to report as available (for testing)
|
||||
- False: force client to report as unavailable
|
||||
"""
|
||||
self._force_available = force_available
|
||||
|
||||
@property
|
||||
def is_available(self) -> bool:
|
||||
"""Check if the Kiro agent is currently available.
|
||||
|
||||
Detection strategy:
|
||||
1. If force_available is set explicitly, use that value.
|
||||
2. Otherwise, check for KIRO_SESSION_ACTIVE env var.
|
||||
3. Default: assume NOT available (safe fallback that triggers
|
||||
enrichment_pending rather than silently skipping enrichment).
|
||||
|
||||
Returns:
|
||||
True if the agent session is active.
|
||||
"""
|
||||
if self._force_available is not None:
|
||||
return self._force_available
|
||||
|
||||
import os
|
||||
|
||||
# Check environment marker set by the Kiro agent runtime
|
||||
session_marker = os.environ.get("KIRO_SESSION_ACTIVE", "").lower()
|
||||
return session_marker in ("1", "true", "yes")
|
||||
|
||||
def enrich_content(self, content: str, source_type: str = "") -> KiroEnrichmentResult:
|
||||
"""Analyze content using the Kiro AI-Agent for enrichment.
|
||||
|
||||
When the agent is available, this method delegates to the Kiro runtime
|
||||
for LLM-based analysis. When unavailable, it raises KiroUnavailableError
|
||||
to signal that the caller should set enrichment_pending: true.
|
||||
|
||||
Args:
|
||||
content: The text content to analyze.
|
||||
source_type: Optional source type hint (e.g., "confluence", "jira").
|
||||
|
||||
Returns:
|
||||
KiroEnrichmentResult with LLM-generated metadata.
|
||||
|
||||
Raises:
|
||||
KiroUnavailableError: If the Kiro agent session is not active.
|
||||
"""
|
||||
if not self.is_available:
|
||||
raise KiroUnavailableError(
|
||||
"Kiro AI-Agent session is not active. "
|
||||
"Set KIRO_SESSION_ACTIVE=1 or run within an active Kiro session."
|
||||
)
|
||||
|
||||
# --- Agent Runtime Invocation Placeholder ---
|
||||
# In a live Kiro session, this is where the agent runtime would be
|
||||
# called to perform LLM-based content analysis. The agent processes
|
||||
# the content and returns structured enrichment data.
|
||||
#
|
||||
# For now, this returns a minimal placeholder result indicating that
|
||||
# the interface is working but the actual LLM call needs the agent
|
||||
# runtime context (which is available during interactive sessions).
|
||||
logger.info(
|
||||
"Kiro enrich_content called (content_length=%d, source_type=%s)",
|
||||
len(content),
|
||||
source_type or "unspecified",
|
||||
)
|
||||
|
||||
return KiroEnrichmentResult(
|
||||
title="",
|
||||
category="inbox",
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
def ocr_image(self, image_path: Path) -> str:
|
||||
"""Extract text from an image using Kiro Vision.
|
||||
|
||||
Sends the image to the Kiro agent which uses its vision capabilities
|
||||
to extract text content. This is the primary OCR method - no external
|
||||
OCR engine (Tesseract) is required when Kiro is active.
|
||||
|
||||
Args:
|
||||
image_path: Path to the image file.
|
||||
|
||||
Returns:
|
||||
Extracted text from the image.
|
||||
|
||||
Raises:
|
||||
KiroUnavailableError: If the Kiro agent is not active.
|
||||
FileNotFoundError: If the image file does not exist.
|
||||
"""
|
||||
if not image_path.exists():
|
||||
raise FileNotFoundError(f"Image file not found: {image_path}")
|
||||
|
||||
if not self.is_available:
|
||||
raise KiroUnavailableError(
|
||||
"Kiro Vision is not available. "
|
||||
"OCR requires an active Kiro agent session."
|
||||
)
|
||||
|
||||
# --- Agent Runtime Invocation Placeholder ---
|
||||
# In a live Kiro session, the image would be sent to the agent's
|
||||
# vision model for text extraction. The agent reads the image and
|
||||
# returns the extracted text.
|
||||
logger.info("Kiro ocr_image called (path=%s)", image_path)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Factory Function
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def create_kiro_client(*, force_available: bool | None = None) -> KiroClient | None:
|
||||
"""Factory function to create a KiroClient if the agent is available.
|
||||
|
||||
This is the recommended way to obtain a KiroClient instance. It checks
|
||||
availability and returns None if Kiro is not active, allowing the caller
|
||||
to handle the fallback gracefully.
|
||||
|
||||
Usage:
|
||||
client = create_kiro_client()
|
||||
if client is None:
|
||||
# Kiro not available → set enrichment_pending: true
|
||||
artifact.enrichment_pending = True
|
||||
else:
|
||||
result = client.enrich_content(content)
|
||||
|
||||
Args:
|
||||
force_available: Override auto-detection (for testing).
|
||||
|
||||
Returns:
|
||||
KiroClient instance if available, None otherwise.
|
||||
"""
|
||||
client = KiroClient(force_available=force_available)
|
||||
|
||||
if not client.is_available:
|
||||
logger.debug("Kiro client not available – enrichment will be marked as pending")
|
||||
return None
|
||||
|
||||
logger.debug("Kiro client is available")
|
||||
return client
|
||||
|
||||
|
||||
def check_kiro_available() -> bool:
|
||||
"""Quick check if Kiro is available without creating a client.
|
||||
|
||||
Useful for conditional logic where you just need to know if enrichment
|
||||
will work, without creating the full client object.
|
||||
|
||||
Returns:
|
||||
True if Kiro agent session is active.
|
||||
"""
|
||||
client = KiroClient()
|
||||
return client.is_available
|
||||
@@ -0,0 +1,326 @@
|
||||
"""OrgMyLife-Integration für das Knowledge-Management.
|
||||
|
||||
Erstellt Tasks in OrgMyLife aus erkannten Action-Items und verwaltet
|
||||
die Deduplication über eine Hash-basierte Mapping-Datei.
|
||||
|
||||
Requirements: 10.1, 10.2, 10.3, 10.4, 10.5, 10.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.ingestion.enrichment import ActionItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskMapping:
|
||||
"""Maps artifact action items to OrgMyLife task IDs.
|
||||
|
||||
Attributes:
|
||||
artifact_id: Identifier of the source artifact.
|
||||
action_item_hash: SHA-256 hash of the action item description.
|
||||
task_id: OrgMyLife task ID returned by the API.
|
||||
created: ISO date string when the mapping was created.
|
||||
"""
|
||||
|
||||
artifact_id: str
|
||||
action_item_hash: str
|
||||
task_id: int
|
||||
created: str # ISO date
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Default filename for the task mapping file.
|
||||
MAPPING_FILENAME = ".task-mapping.yaml"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OrgMyLife Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class OrgMyLifeIntegration:
|
||||
"""Creates tasks in OrgMyLife from detected action items.
|
||||
|
||||
Uses a mapping file (`{context}/knowledge/.task-mapping.yaml`) to track
|
||||
which action items have already been created as tasks, preventing
|
||||
duplicate task creation on repeated ingestion runs.
|
||||
|
||||
The actual OrgMyLife API call is stubbed with a TODO for now.
|
||||
The integration handles API failures gracefully by logging warnings
|
||||
and continuing without task creation.
|
||||
|
||||
Attributes:
|
||||
context_path: Path to the context knowledge folder.
|
||||
enabled: Whether task creation is enabled.
|
||||
"""
|
||||
|
||||
def __init__(self, context_path: Path, enabled: bool = True) -> None:
|
||||
"""Initialize the OrgMyLife integration.
|
||||
|
||||
Args:
|
||||
context_path: Path to the context knowledge folder
|
||||
(e.g., `{root}/bahn/knowledge/`).
|
||||
enabled: Whether task creation is active. If False,
|
||||
create_tasks() returns an empty list.
|
||||
"""
|
||||
self.context_path = context_path
|
||||
self.enabled = enabled
|
||||
self._mapping_path = context_path / MAPPING_FILENAME
|
||||
|
||||
def create_tasks(
|
||||
self,
|
||||
artifact_id: str,
|
||||
action_items: list[ActionItem],
|
||||
source_path: str,
|
||||
) -> list[TaskMapping]:
|
||||
"""Create OrgMyLife tasks for new action items (dedup via mapping file).
|
||||
|
||||
Only creates tasks for action items whose hash is not yet present
|
||||
in the mapping file. If the OrgMyLife API is unavailable, logs a
|
||||
warning and returns an empty list.
|
||||
|
||||
Args:
|
||||
artifact_id: Unique identifier of the source artifact.
|
||||
action_items: List of ActionItem instances to create tasks for.
|
||||
source_path: Path to the source artifact file (used as source_url).
|
||||
|
||||
Returns:
|
||||
List of TaskMapping instances for newly created tasks.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return []
|
||||
|
||||
if not action_items:
|
||||
return []
|
||||
|
||||
# Load existing mappings for deduplication
|
||||
existing_mappings = self._load_mapping()
|
||||
|
||||
new_mappings: list[TaskMapping] = []
|
||||
|
||||
for item in action_items:
|
||||
# Build dedup key: artifact_id + action_item_hash
|
||||
dedup_key = f"{artifact_id}:{item.hash}"
|
||||
|
||||
# Skip if already mapped (Requirement 10.5, 10.6)
|
||||
if dedup_key in existing_mappings:
|
||||
logger.debug(
|
||||
"Action item already mapped: %s (task_id=%d)",
|
||||
dedup_key,
|
||||
existing_mappings[dedup_key].task_id,
|
||||
)
|
||||
continue
|
||||
|
||||
# Create task via OrgMyLife API
|
||||
task_id = self._create_orgmylife_task(
|
||||
title=item.description,
|
||||
source_url=source_path,
|
||||
deadline=item.deadline,
|
||||
)
|
||||
|
||||
if task_id is None:
|
||||
# API failure – logged in _create_orgmylife_task, continue
|
||||
continue
|
||||
|
||||
# Record new mapping
|
||||
mapping = TaskMapping(
|
||||
artifact_id=artifact_id,
|
||||
action_item_hash=item.hash,
|
||||
task_id=task_id,
|
||||
created=date.today().isoformat(),
|
||||
)
|
||||
new_mappings.append(mapping)
|
||||
existing_mappings[dedup_key] = mapping
|
||||
|
||||
# Persist updated mappings
|
||||
if new_mappings:
|
||||
self._save_mapping(existing_mappings)
|
||||
logger.info(
|
||||
"Created %d OrgMyLife task(s) for artifact '%s'",
|
||||
len(new_mappings),
|
||||
artifact_id,
|
||||
)
|
||||
|
||||
return new_mappings
|
||||
|
||||
def _load_mapping(self) -> dict[str, TaskMapping]:
|
||||
"""Load task mappings from the YAML file.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping dedup keys (artifact_id:hash) to TaskMapping.
|
||||
"""
|
||||
if not self._mapping_path.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
raw = yaml.safe_load(self._mapping_path.read_text(encoding="utf-8"))
|
||||
except (yaml.YAMLError, OSError) as exc:
|
||||
logger.warning(
|
||||
"Failed to load task mapping from %s: %s",
|
||||
self._mapping_path,
|
||||
exc,
|
||||
)
|
||||
return {}
|
||||
|
||||
if not raw or not isinstance(raw, dict):
|
||||
return {}
|
||||
|
||||
mappings: dict[str, TaskMapping] = {}
|
||||
entries = raw.get("mappings", [])
|
||||
|
||||
if not isinstance(entries, list):
|
||||
return {}
|
||||
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
try:
|
||||
mapping = TaskMapping(
|
||||
artifact_id=str(entry["artifact_id"]),
|
||||
action_item_hash=str(entry["action_item_hash"]),
|
||||
task_id=int(entry["task_id"]),
|
||||
created=str(entry.get("created", "")),
|
||||
)
|
||||
dedup_key = f"{mapping.artifact_id}:{mapping.action_item_hash}"
|
||||
mappings[dedup_key] = mapping
|
||||
except (KeyError, ValueError, TypeError) as exc:
|
||||
logger.warning("Skipping invalid mapping entry: %s", exc)
|
||||
continue
|
||||
|
||||
return mappings
|
||||
|
||||
def _save_mapping(self, mappings: dict[str, TaskMapping]) -> None:
|
||||
"""Save task mappings to the YAML file.
|
||||
|
||||
Args:
|
||||
mappings: Dictionary of dedup keys to TaskMapping instances.
|
||||
"""
|
||||
entries: list[dict[str, Any]] = []
|
||||
for mapping in mappings.values():
|
||||
entries.append({
|
||||
"artifact_id": mapping.artifact_id,
|
||||
"action_item_hash": mapping.action_item_hash,
|
||||
"task_id": mapping.task_id,
|
||||
"created": mapping.created,
|
||||
})
|
||||
|
||||
data = {
|
||||
"version": "1.0",
|
||||
"mappings": entries,
|
||||
}
|
||||
|
||||
try:
|
||||
self._mapping_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._mapping_path.write_text(
|
||||
yaml.dump(data, default_flow_style=False, allow_unicode=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to save task mapping to %s: %s",
|
||||
self._mapping_path,
|
||||
exc,
|
||||
)
|
||||
|
||||
def _create_orgmylife_task(
|
||||
self,
|
||||
title: str,
|
||||
source_url: str,
|
||||
deadline: str | None = None,
|
||||
) -> int | None:
|
||||
"""Create a task in OrgMyLife via the REST API.
|
||||
|
||||
This method handles API errors gracefully: on failure it logs a
|
||||
warning and returns None (Requirement 10.4).
|
||||
|
||||
Args:
|
||||
title: Task title (action item description).
|
||||
source_url: Reference to the source artifact.
|
||||
deadline: Optional ISO date string for the task deadline.
|
||||
|
||||
Returns:
|
||||
The OrgMyLife task ID on success, or None on failure.
|
||||
"""
|
||||
# TODO: Implement actual OrgMyLife REST API call.
|
||||
# The OrgMyLife API endpoint for task creation:
|
||||
# POST /api/tasks
|
||||
# Body: {"title": title, "repo": "", "deadline": deadline}
|
||||
# Response: {"id": <int>, "title": ..., ...}
|
||||
#
|
||||
# For now, use a stub that simulates a successful API call.
|
||||
# In production, replace with actual HTTP client (httpx/requests).
|
||||
|
||||
try:
|
||||
task_id = self._call_orgmylife_api(title, source_url, deadline)
|
||||
return task_id
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Graceful error handling (Requirement 10.4)
|
||||
logger.warning(
|
||||
"OrgMyLife API error creating task '%s': %s",
|
||||
title,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
def _call_orgmylife_api(
|
||||
self,
|
||||
title: str,
|
||||
source_url: str,
|
||||
deadline: str | None = None,
|
||||
) -> int:
|
||||
"""Actually call the OrgMyLife API.
|
||||
|
||||
This is separated from _create_orgmylife_task to allow easy
|
||||
mocking in tests and future replacement with real HTTP calls.
|
||||
|
||||
Args:
|
||||
title: Task title.
|
||||
source_url: Reference to source artifact.
|
||||
deadline: Optional deadline ISO date string.
|
||||
|
||||
Returns:
|
||||
The created task ID.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the API call fails.
|
||||
"""
|
||||
# TODO: Replace with actual HTTP call to OrgMyLife API.
|
||||
# Example implementation with httpx:
|
||||
#
|
||||
# import httpx
|
||||
# response = httpx.post(
|
||||
# f"{ORGMYLIFE_BASE_URL}/api/tasks",
|
||||
# json={
|
||||
# "title": title,
|
||||
# "repo": "",
|
||||
# "source_url": source_url,
|
||||
# "deadline": deadline,
|
||||
# },
|
||||
# timeout=10.0,
|
||||
# )
|
||||
# response.raise_for_status()
|
||||
# return response.json()["id"]
|
||||
|
||||
# Stub: raise to signal that real API is not implemented yet.
|
||||
# In tests, this method is patched to return a task ID.
|
||||
raise NotImplementedError(
|
||||
"OrgMyLife API not yet connected. "
|
||||
"Patch _call_orgmylife_api in tests or implement HTTP client."
|
||||
)
|
||||
@@ -4,8 +4,32 @@ Jede Quellstrategie implementiert das Extrahieren von Wissensartefakten
|
||||
aus einem bestimmten Format/Medium (Markdown, Confluence, PDF, etc.).
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
from monorepo.knowledge.sources.chat import ChatSource
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource
|
||||
from monorepo.knowledge.sources.email import EmailSource
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.link import LinkRegistry
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.knowledge.sources.pdf import PDFSource
|
||||
from monorepo.knowledge.sources.wissensdatenbank import WissensdatenbankSource
|
||||
|
||||
__all__ = [
|
||||
"ChatSource",
|
||||
"ConfluenceSource",
|
||||
"EmailSource",
|
||||
"ExtractionResult",
|
||||
"JiraSource",
|
||||
"LinkRegistry",
|
||||
"MarkdownSource",
|
||||
"PDFSource",
|
||||
"SourceConfig",
|
||||
"SourceError",
|
||||
"SourceStrategy",
|
||||
"WissensdatenbankSource",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Abstrakte Basisklasse für Quellstrategien der ETL-Pipeline.
|
||||
|
||||
Definiert das SourceStrategy-Interface, das alle konkreten Quellstrategien
|
||||
(Markdown, PDF, Confluence, Jira, etc.) implementieren müssen.
|
||||
|
||||
Requirements: 2.2, 2.4, 2.7, 4.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Format Detection (Requirement 4.7)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Mapping von unterstützten Dateiendungen (ohne Punkt, lowercase) auf Formattypen.
|
||||
EXTENSION_FORMAT_MAP: dict[str, str] = {
|
||||
"md": "markdown",
|
||||
"txt": "text",
|
||||
"pdf": "pdf",
|
||||
"png": "image",
|
||||
"jpg": "image",
|
||||
"jpeg": "image",
|
||||
"docx": "docx",
|
||||
}
|
||||
|
||||
#: Menge aller unterstützten Endungen (lowercase, ohne Punkt).
|
||||
SUPPORTED_EXTENSIONS: set[str] = set(EXTENSION_FORMAT_MAP.keys())
|
||||
|
||||
|
||||
def detect_format(file_path: str) -> str:
|
||||
"""Erkennt den Formattyp anhand der Dateiendung.
|
||||
|
||||
Unterstützte Zuordnungen:
|
||||
- .md → "markdown"
|
||||
- .txt → "text"
|
||||
- .pdf → "pdf"
|
||||
- .png, .jpg, .jpeg → "image"
|
||||
- .docx → "docx"
|
||||
|
||||
Args:
|
||||
file_path: Ein Dateipfad (kann relativ oder absolut sein).
|
||||
|
||||
Returns:
|
||||
Der erkannte Formattyp als String.
|
||||
|
||||
Raises:
|
||||
ValueError: Falls die Dateiendung nicht unterstützt wird.
|
||||
"""
|
||||
# Support both Unix and Windows paths
|
||||
# Use suffix extraction that handles both separators
|
||||
path_str = file_path.strip()
|
||||
if not path_str:
|
||||
raise ValueError("Empty file path")
|
||||
|
||||
# Extract extension: find the last '.' in the filename portion
|
||||
# Handle both forward and backslash separators
|
||||
filename = path_str.replace("\\", "/").rsplit("/", 1)[-1]
|
||||
if "." not in filename or filename.startswith("."):
|
||||
raise ValueError(
|
||||
f"Unsupported or missing file extension in: {file_path!r}"
|
||||
)
|
||||
|
||||
ext = filename.rsplit(".", 1)[-1].lower()
|
||||
|
||||
if ext in EXTENSION_FORMAT_MAP:
|
||||
return EXTENSION_FORMAT_MAP[ext]
|
||||
|
||||
raise ValueError(
|
||||
f"Unsupported file extension '.{ext}' in: {file_path!r}. "
|
||||
f"Supported: {', '.join('.' + e for e in sorted(SUPPORTED_EXTENSIONS))}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Konfiguration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceConfig:
|
||||
"""Konfiguration für eine einzelne Quelle in sources.yaml.
|
||||
|
||||
Attributes:
|
||||
type: Quelltyp (confluence, jira, email, file, link, wissensdatenbank).
|
||||
name: Menschenlesbare Bezeichnung der Quelle.
|
||||
params: Typspezifische Verbindungs-/Filterparameter.
|
||||
target_folder: Optionaler Override für den Zielordner.
|
||||
sync_frequency: Synchronisierungsfrequenz (on-push, hourly, daily, manual).
|
||||
enabled: Ob die Quelle aktiv ist.
|
||||
"""
|
||||
|
||||
type: str
|
||||
name: str
|
||||
params: dict[str, Any] = field(default_factory=dict)
|
||||
target_folder: str = ""
|
||||
sync_frequency: str = "manual"
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ergebnisse und Fehler
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceError:
|
||||
"""Strukturierter Fehler bei der Quellextraktion.
|
||||
|
||||
Attributes:
|
||||
source_name: Bezeichnung der betroffenen Quelle.
|
||||
error_type: Fehlerkategorie (connection, auth, parse, write, llm).
|
||||
message: Menschenlesbare Fehlerbeschreibung.
|
||||
timestamp: Zeitstempel im ISO-8601-Format.
|
||||
retry: Ob ein Wiederholungsversuch sinnvoll ist.
|
||||
artifact_path: Betroffener Artefaktpfad, falls zutreffend.
|
||||
"""
|
||||
|
||||
source_name: str
|
||||
error_type: str
|
||||
message: str
|
||||
timestamp: str
|
||||
retry: bool
|
||||
artifact_path: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionResult:
|
||||
"""Ergebnis einer Quellstrategie-Extraktion.
|
||||
|
||||
Attributes:
|
||||
artifacts: Erfolgreich extrahierte Artefakte.
|
||||
errors: Fehler, die bei der Extraktion aufgetreten sind.
|
||||
skipped: Anzahl übersprungener Elemente (z.B. unveränderte Dateien).
|
||||
"""
|
||||
|
||||
artifacts: list[Artifact] = field(default_factory=list)
|
||||
errors: list[SourceError] = field(default_factory=list)
|
||||
skipped: int = 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Abstrakte Basisklasse
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SourceStrategy(ABC):
|
||||
"""Abstrakte Basisklasse für alle Quellstrategien.
|
||||
|
||||
Jede konkrete Quellstrategie (Markdown, PDF, Confluence, etc.) muss
|
||||
dieses Interface implementieren, um in die Ingestion-Pipeline integriert
|
||||
werden zu können.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
|
||||
"""Extrahiert Rohartefakte aus der konfigurierten Quelle.
|
||||
|
||||
Args:
|
||||
config: Die Quellkonfiguration mit Verbindungsparametern.
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Gibt an, ob diese Strategie inkrementelle Verarbeitung unterstützt.
|
||||
|
||||
Bei inkrementeller Verarbeitung werden nur neue oder geänderte
|
||||
Inhalte verarbeitet (z.B. über Content-Hashes).
|
||||
|
||||
Returns:
|
||||
True wenn inkrementelle Verarbeitung unterstützt wird.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,365 @@
|
||||
"""Chat-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Verarbeitet Chat-Exporte in Text- oder JSON-Format und erstellt
|
||||
strukturierte Markdown-Artefakte mit Frontmatter.
|
||||
|
||||
Unterstützte Formate:
|
||||
- Plain-Text mit Timestamp-Patterns (z.B. "[2025-01-15 14:30] User: Message")
|
||||
- JSON-Arrays mit Nachrichtenobjekten (sender, timestamp, text)
|
||||
|
||||
Requirements: 7.3, 7.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["ChatSource"]
|
||||
|
||||
# Supported chat export file extensions
|
||||
_CHAT_EXTENSIONS = {".txt", ".json"}
|
||||
|
||||
# Patterns for detecting timestamped chat messages in plain text
|
||||
# Format: [YYYY-MM-DD HH:MM] or [YYYY-MM-DD HH:MM:SS] followed by sender: message
|
||||
_TIMESTAMP_PATTERNS = [
|
||||
# [2025-01-15 14:30] User: Message
|
||||
re.compile(
|
||||
r"^\[(\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}(?::\d{2})?)\]\s*(.+?):\s*(.+)$",
|
||||
re.MULTILINE,
|
||||
),
|
||||
# 2025-01-15 14:30 - User: Message (WhatsApp-style)
|
||||
re.compile(
|
||||
r"^(\d{4}-\d{2}-\d{2}[\s,]+\d{2}:\d{2}(?::\d{2})?)\s*-\s*(.+?):\s*(.+)$",
|
||||
re.MULTILINE,
|
||||
),
|
||||
# DD.MM.YYYY, HH:MM - User: Message (German WhatsApp)
|
||||
re.compile(
|
||||
r"^(\d{2}\.\d{2}\.\d{4},?\s*\d{2}:\d{2}(?::\d{2})?)\s*-\s*(.+?):\s*(.+)$",
|
||||
re.MULTILINE,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Creates a URL-friendly slug from text."""
|
||||
slug = text.lower().strip()
|
||||
slug = re.sub(r"[^a-z0-9äöüß\s-]", "", slug)
|
||||
slug = re.sub(r"[\s_]+", "-", slug)
|
||||
slug = re.sub(r"-+", "-", slug)
|
||||
slug = slug.strip("-")
|
||||
return slug[:80] if slug else "chat-export"
|
||||
|
||||
|
||||
def _compute_content_hash(content: str) -> str:
|
||||
"""Computes SHA-256 hash of content."""
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
def _parse_date_str(date_str: str) -> datetime | None:
|
||||
"""Attempts to parse various date formats into a datetime."""
|
||||
formats = [
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%dT%H:%M",
|
||||
"%d.%m.%Y, %H:%M:%S",
|
||||
"%d.%m.%Y, %H:%M",
|
||||
"%d.%m.%Y %H:%M:%S",
|
||||
"%d.%m.%Y %H:%M",
|
||||
]
|
||||
date_str = date_str.strip()
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(date_str, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatMessage:
|
||||
"""A single parsed chat message."""
|
||||
|
||||
timestamp: datetime | None
|
||||
sender: str
|
||||
text: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ChatSource – implementiert SourceStrategy ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ChatSource(SourceStrategy):
|
||||
"""Quellstrategie: Verarbeitet Chat-Exporte (Text/JSON) aus einem Verzeichnis.
|
||||
|
||||
Unterstützt:
|
||||
- Plain-Text-Exporte mit erkennbaren Timestamp-Patterns
|
||||
- JSON-Exporte als Array von Message-Objekten
|
||||
|
||||
Erstellt pro Exportdatei ein Markdown-Artefakt mit Konversationsinhalt
|
||||
und Frontmatter-Metadaten (Teilnehmer, Zeitraum, Quelltyp).
|
||||
"""
|
||||
|
||||
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
|
||||
"""Extrahiert Chat-Artefakte aus dem konfigurierten Verzeichnis.
|
||||
|
||||
Args:
|
||||
config: Quellkonfiguration mit params["directory"] als Pfad.
|
||||
context: Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und Fehlern.
|
||||
"""
|
||||
directory = self._resolve_directory(config)
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
|
||||
if not directory.exists():
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Verzeichnis existiert nicht: {directory}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
))
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
if not directory.is_dir():
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Pfad ist kein Verzeichnis: {directory}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
))
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
# Find all chat export files
|
||||
chat_files = sorted(
|
||||
f for f in directory.rglob("*")
|
||||
if f.is_file() and f.suffix.lower() in _CHAT_EXTENSIONS
|
||||
)
|
||||
|
||||
for file_path in chat_files:
|
||||
try:
|
||||
artifact = self._process_chat_file(file_path, config.name)
|
||||
if artifact is not None:
|
||||
artifacts.append(artifact)
|
||||
except Exception as e:
|
||||
logger.warning("Fehler beim Verarbeiten von %s: %s", file_path, e)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler bei {file_path.name}: {e}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
artifact_path=str(file_path),
|
||||
))
|
||||
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Chat-Exporte unterstützen keine inkrementelle Verarbeitung."""
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_directory(self, config: SourceConfig) -> Path:
|
||||
"""Resolves the source directory from config."""
|
||||
dir_param = config.params.get("directory", "")
|
||||
if dir_param:
|
||||
return Path(dir_param)
|
||||
raise ValueError(
|
||||
"ChatSource benötigt params['directory'] in der Konfiguration."
|
||||
)
|
||||
|
||||
def _process_chat_file(self, file_path: Path, source_name: str) -> Artifact | None:
|
||||
"""Processes a single chat export file."""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix == ".json":
|
||||
return self._process_json_chat(file_path, source_name)
|
||||
elif suffix == ".txt":
|
||||
return self._process_text_chat(file_path, source_name)
|
||||
return None
|
||||
|
||||
def _process_json_chat(self, file_path: Path, source_name: str) -> Artifact | None:
|
||||
"""Processes a JSON chat export.
|
||||
|
||||
Expected format: array of objects with keys:
|
||||
- sender (str): message author
|
||||
- timestamp (str): ISO-format or similar date string
|
||||
- text (str): message content
|
||||
"""
|
||||
raw_text = file_path.read_text(encoding="utf-8")
|
||||
data = json.loads(raw_text)
|
||||
|
||||
if not isinstance(data, list):
|
||||
logger.warning("JSON-Chat %s ist kein Array, überspringe.", file_path)
|
||||
return None
|
||||
|
||||
messages: list[ChatMessage] = []
|
||||
for entry in data:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
sender = str(entry.get("sender", entry.get("author", entry.get("from", ""))))
|
||||
text = str(entry.get("text", entry.get("message", entry.get("content", ""))))
|
||||
ts_str = str(entry.get("timestamp", entry.get("date", entry.get("time", ""))))
|
||||
timestamp = _parse_date_str(ts_str) if ts_str else None
|
||||
if text:
|
||||
messages.append(ChatMessage(timestamp=timestamp, sender=sender, text=text))
|
||||
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
return self._build_artifact(file_path, messages, source_name, "json")
|
||||
|
||||
def _process_text_chat(self, file_path: Path, source_name: str) -> Artifact | None:
|
||||
"""Processes a plain-text chat export with timestamp patterns."""
|
||||
raw_text = file_path.read_text(encoding="utf-8")
|
||||
|
||||
messages: list[ChatMessage] = []
|
||||
|
||||
# Try each pattern until one matches
|
||||
for pattern in _TIMESTAMP_PATTERNS:
|
||||
matches = pattern.findall(raw_text)
|
||||
if matches:
|
||||
for match in matches:
|
||||
ts_str, sender, text = match
|
||||
timestamp = _parse_date_str(ts_str)
|
||||
messages.append(ChatMessage(
|
||||
timestamp=timestamp,
|
||||
sender=sender.strip(),
|
||||
text=text.strip(),
|
||||
))
|
||||
break
|
||||
|
||||
# If no pattern matched, treat entire file as a single message block
|
||||
if not messages:
|
||||
# Still create an artifact from the raw content
|
||||
messages.append(ChatMessage(
|
||||
timestamp=None,
|
||||
sender="unknown",
|
||||
text=raw_text.strip(),
|
||||
))
|
||||
|
||||
return self._build_artifact(file_path, messages, source_name, "text")
|
||||
|
||||
def _build_artifact(
|
||||
self,
|
||||
file_path: Path,
|
||||
messages: list[ChatMessage],
|
||||
source_name: str,
|
||||
source_format: str,
|
||||
) -> Artifact:
|
||||
"""Builds a Markdown artifact from parsed chat messages."""
|
||||
# Collect participants and date range
|
||||
participants: set[str] = set()
|
||||
dates: list[datetime] = []
|
||||
|
||||
for msg in messages:
|
||||
if msg.sender and msg.sender != "unknown":
|
||||
participants.add(msg.sender)
|
||||
if msg.timestamp:
|
||||
dates.append(msg.timestamp)
|
||||
|
||||
date_start = min(dates).date() if dates else None
|
||||
date_end = max(dates).date() if dates else None
|
||||
|
||||
# Build Markdown content
|
||||
title = f"Chat: {file_path.stem}"
|
||||
content_lines: list[str] = []
|
||||
content_lines.append(f"# {title}\n")
|
||||
content_lines.append("")
|
||||
|
||||
if participants:
|
||||
content_lines.append(f"**Teilnehmer:** {', '.join(sorted(participants))} ")
|
||||
if date_start and date_end:
|
||||
if date_start == date_end:
|
||||
content_lines.append(f"**Datum:** {date_start.isoformat()} ")
|
||||
else:
|
||||
content_lines.append(
|
||||
f"**Zeitraum:** {date_start.isoformat()} bis {date_end.isoformat()} "
|
||||
)
|
||||
content_lines.append("")
|
||||
content_lines.append("---\n")
|
||||
content_lines.append("")
|
||||
|
||||
# Render messages
|
||||
for msg in messages:
|
||||
ts_prefix = ""
|
||||
if msg.timestamp:
|
||||
ts_prefix = f"[{msg.timestamp.strftime('%Y-%m-%d %H:%M')}] "
|
||||
sender_prefix = f"**{msg.sender}:** " if msg.sender and msg.sender != "unknown" else ""
|
||||
content_lines.append(f"{ts_prefix}{sender_prefix}{msg.text} ")
|
||||
|
||||
content = "\n".join(content_lines)
|
||||
|
||||
# Build source dict for frontmatter
|
||||
source_dict: dict[str, str] = {
|
||||
"type": "chat",
|
||||
"file": str(file_path),
|
||||
"format": source_format,
|
||||
}
|
||||
|
||||
# Date range for frontmatter
|
||||
date_range = ""
|
||||
if date_start and date_end:
|
||||
date_range = f"{date_start.isoformat()}/{date_end.isoformat()}"
|
||||
elif date_start:
|
||||
date_range = date_start.isoformat()
|
||||
|
||||
if date_range:
|
||||
source_dict["date_range"] = date_range
|
||||
|
||||
# Build metadata
|
||||
metadata = ArtifactMetadata(
|
||||
type="chat",
|
||||
title=title,
|
||||
tags=["chat"],
|
||||
source_context="",
|
||||
created=date_start,
|
||||
updated=date_end,
|
||||
shareable=False,
|
||||
source=source_dict,
|
||||
category="inbox",
|
||||
people=sorted(participants),
|
||||
content_hash=_compute_content_hash(content),
|
||||
)
|
||||
|
||||
# Generate filename
|
||||
date_prefix = date_start.isoformat() if date_start else "undated"
|
||||
slug = _slugify(file_path.stem)
|
||||
filename = f"{date_prefix}-{slug}.md"
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=Path(filename),
|
||||
)
|
||||
@@ -0,0 +1,686 @@
|
||||
"""Confluence-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Implementiert die Anbindung an die Confluence REST API (Cloud und Server):
|
||||
- Einzelne Seiten abrufen
|
||||
- Seitenbäume rekursiv traversieren
|
||||
- FAQ-Spaces importieren
|
||||
- Inkrementelle Updates via Versions-Nummer
|
||||
- Graceful Error Handling (skip source, log error)
|
||||
|
||||
Seiteninhalte werden als Markdown extrahiert und mit reichhaltigem
|
||||
Frontmatter angereichert (URL, Space, Titel, Last-Modified).
|
||||
|
||||
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from base64 import b64encode
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTML → Markdown Konvertierung (leichtgewichtig, kein externes Paket)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _html_to_markdown(html: str) -> str:
|
||||
"""Konvertiert einfaches HTML in Markdown.
|
||||
|
||||
Unterstützt die gängigsten Confluence-HTML-Elemente:
|
||||
- Überschriften (h1-h6)
|
||||
- Absätze, Zeilenumbrüche
|
||||
- Listen (ul/ol/li)
|
||||
- Fett/Kursiv
|
||||
- Links
|
||||
- Code-Blöcke und Inline-Code
|
||||
- Tabellen (vereinfacht)
|
||||
|
||||
Args:
|
||||
html: HTML-String aus Confluence Storage Format.
|
||||
|
||||
Returns:
|
||||
Markdown-String.
|
||||
"""
|
||||
if not html:
|
||||
return ""
|
||||
|
||||
text = html
|
||||
|
||||
# Entferne CDATA und XML-Deklarationen
|
||||
text = re.sub(r"<!\[CDATA\[.*?\]\]>", "", text, flags=re.DOTALL)
|
||||
|
||||
# Code-Blöcke zuerst (vor allgemeinem Tag-Stripping)
|
||||
text = re.sub(
|
||||
r"<ac:structured-macro[^>]*ac:name=\"code\"[^>]*>.*?"
|
||||
r"<ac:plain-text-body>(.*?)</ac:plain-text-body>.*?"
|
||||
r"</ac:structured-macro>",
|
||||
r"\n```\n\1\n```\n",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
|
||||
# Confluence-spezifische Macros entfernen
|
||||
text = re.sub(
|
||||
r"<ac:structured-macro[^>]*>.*?</ac:structured-macro>",
|
||||
"",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
text = re.sub(r"<ac:[^>]*/>", "", text)
|
||||
text = re.sub(r"<ac:[^>]*>.*?</ac:[^>]*>", "", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<ri:[^>]*/>", "", text)
|
||||
text = re.sub(r"<ri:[^>]*>.*?</ri:[^>]*>", "", text, flags=re.DOTALL)
|
||||
|
||||
# Überschriften
|
||||
for level in range(1, 7):
|
||||
prefix = "#" * level
|
||||
text = re.sub(
|
||||
rf"<h{level}[^>]*>(.*?)</h{level}>",
|
||||
rf"\n{prefix} \1\n",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
|
||||
# Links
|
||||
text = re.sub(
|
||||
r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>',
|
||||
r"[\2](\1)",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
|
||||
# Fett und Kursiv
|
||||
text = re.sub(r"<strong[^>]*>(.*?)</strong>", r"**\1**", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<b[^>]*>(.*?)</b>", r"**\1**", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<em[^>]*>(.*?)</em>", r"*\1*", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<i[^>]*>(.*?)</i>", r"*\1*", text, flags=re.DOTALL)
|
||||
|
||||
# Code (inline)
|
||||
text = re.sub(r"<code[^>]*>(.*?)</code>", r"`\1`", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<pre[^>]*>(.*?)</pre>", r"\n```\n\1\n```\n", text, flags=re.DOTALL)
|
||||
|
||||
# Listen
|
||||
text = re.sub(r"<ul[^>]*>", "\n", text)
|
||||
text = re.sub(r"</ul>", "\n", text)
|
||||
text = re.sub(r"<ol[^>]*>", "\n", text)
|
||||
text = re.sub(r"</ol>", "\n", text)
|
||||
text = re.sub(r"<li[^>]*>(.*?)</li>", r"- \1\n", text, flags=re.DOTALL)
|
||||
|
||||
# Absätze und Zeilenumbrüche
|
||||
text = re.sub(r"<br\s*/?>", "\n", text)
|
||||
text = re.sub(r"<p[^>]*>(.*?)</p>", r"\1\n\n", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<div[^>]*>(.*?)</div>", r"\1\n", text, flags=re.DOTALL)
|
||||
|
||||
# Tabellen (vereinfachtes Markdown)
|
||||
text = re.sub(r"<table[^>]*>", "\n", text)
|
||||
text = re.sub(r"</table>", "\n", text)
|
||||
text = re.sub(r"<thead[^>]*>|</thead>", "", text)
|
||||
text = re.sub(r"<tbody[^>]*>|</tbody>", "", text)
|
||||
text = re.sub(r"<tr[^>]*>(.*?)</tr>", r"|\1\n", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<th[^>]*>(.*?)</th>", r" \1 |", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<td[^>]*>(.*?)</td>", r" \1 |", text, flags=re.DOTALL)
|
||||
|
||||
# Alle verbleibenden HTML-Tags entfernen
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
|
||||
# HTML-Entities dekodieren
|
||||
text = unescape(text)
|
||||
|
||||
# Mehrfache Leerzeilen reduzieren
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
|
||||
return text.strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Confluence API Client
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConfluencePage:
|
||||
"""Repräsentiert eine einzelne Confluence-Seite."""
|
||||
|
||||
page_id: str
|
||||
title: str
|
||||
space_key: str
|
||||
version: int
|
||||
last_modified: str
|
||||
content_html: str
|
||||
url: str
|
||||
attachments: list[dict[str, str]]
|
||||
|
||||
|
||||
class ConfluenceClient:
|
||||
"""Leichtgewichtiger Client für die Confluence REST API.
|
||||
|
||||
Verwendet urllib aus der Standardbibliothek (keine requests-Abhängigkeit).
|
||||
Unterstützt Token-basierte Authentifizierung (Bearer oder Basic mit API-Token).
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str, token: str, username: str = "") -> None:
|
||||
"""Initialisiert den Confluence-Client.
|
||||
|
||||
Args:
|
||||
base_url: Basis-URL der Confluence-Instanz (z.B. https://confluence.example.com).
|
||||
token: API-Token für die Authentifizierung.
|
||||
username: Benutzername für Basic-Auth (optional, für Cloud).
|
||||
"""
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.token = token
|
||||
self.username = username
|
||||
|
||||
def _build_auth_header(self) -> str:
|
||||
"""Erstellt den Authorization-Header.
|
||||
|
||||
Returns:
|
||||
Authorization-Header-Wert.
|
||||
"""
|
||||
if self.username:
|
||||
# Cloud: Basic Auth mit E-Mail + API-Token
|
||||
credentials = f"{self.username}:{self.token}"
|
||||
encoded = b64encode(credentials.encode()).decode()
|
||||
return f"Basic {encoded}"
|
||||
# Server/DC: Bearer Token
|
||||
return f"Bearer {self.token}"
|
||||
|
||||
def _request(self, path: str) -> dict[str, Any]:
|
||||
"""Führt einen GET-Request gegen die Confluence REST API durch.
|
||||
|
||||
Args:
|
||||
path: API-Pfad (relativ zu base_url).
|
||||
|
||||
Returns:
|
||||
JSON-Response als Dictionary.
|
||||
|
||||
Raises:
|
||||
urllib.error.URLError: Bei Netzwerkfehlern.
|
||||
urllib.error.HTTPError: Bei HTTP-Fehlern (401, 403, 404, etc.).
|
||||
"""
|
||||
url = f"{self.base_url}{path}"
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Authorization", self._build_auth_header())
|
||||
req.add_header("Accept", "application/json")
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
def get_page(self, page_id: str) -> ConfluencePage:
|
||||
"""Ruft eine einzelne Confluence-Seite ab.
|
||||
|
||||
Args:
|
||||
page_id: Die ID der Seite.
|
||||
|
||||
Returns:
|
||||
ConfluencePage mit Inhalt und Metadaten.
|
||||
"""
|
||||
path = (
|
||||
f"/rest/api/content/{page_id}"
|
||||
f"?expand=body.storage,version,space,metadata.labels"
|
||||
f"&expand=children.attachment"
|
||||
)
|
||||
data = self._request(path)
|
||||
return self._parse_page_response(data)
|
||||
|
||||
def get_child_pages(self, page_id: str) -> list[str]:
|
||||
"""Ruft die IDs aller direkten Kindseiten ab.
|
||||
|
||||
Args:
|
||||
page_id: Die ID der Elternseite.
|
||||
|
||||
Returns:
|
||||
Liste von Seiten-IDs.
|
||||
"""
|
||||
path = f"/rest/api/content/{page_id}/child/page?limit=200"
|
||||
data = self._request(path)
|
||||
results = data.get("results", [])
|
||||
return [str(r["id"]) for r in results]
|
||||
|
||||
def get_space_pages(self, space_key: str) -> list[str]:
|
||||
"""Ruft alle Seiten-IDs eines Spaces ab.
|
||||
|
||||
Args:
|
||||
space_key: Der Space-Key (z.B. "ACV2").
|
||||
|
||||
Returns:
|
||||
Liste von Seiten-IDs.
|
||||
"""
|
||||
page_ids: list[str] = []
|
||||
start = 0
|
||||
limit = 50
|
||||
|
||||
while True:
|
||||
path = (
|
||||
f"/rest/api/content"
|
||||
f"?spaceKey={space_key}&type=page"
|
||||
f"&start={start}&limit={limit}"
|
||||
)
|
||||
data = self._request(path)
|
||||
results = data.get("results", [])
|
||||
page_ids.extend(str(r["id"]) for r in results)
|
||||
|
||||
# Paginierung
|
||||
size = data.get("size", 0)
|
||||
if size < limit:
|
||||
break
|
||||
start += limit
|
||||
|
||||
return page_ids
|
||||
|
||||
def get_page_tree(self, root_page_id: str) -> list[str]:
|
||||
"""Traversiert einen Seitenbaum rekursiv und sammelt alle Seiten-IDs.
|
||||
|
||||
Args:
|
||||
root_page_id: Die ID der Wurzelseite.
|
||||
|
||||
Returns:
|
||||
Liste aller Seiten-IDs im Baum (inklusive Wurzel).
|
||||
"""
|
||||
collected: list[str] = [root_page_id]
|
||||
queue = [root_page_id]
|
||||
|
||||
while queue:
|
||||
current_id = queue.pop(0)
|
||||
children = self.get_child_pages(current_id)
|
||||
for child_id in children:
|
||||
if child_id not in collected:
|
||||
collected.append(child_id)
|
||||
queue.append(child_id)
|
||||
|
||||
return collected
|
||||
|
||||
def _parse_page_response(self, data: dict[str, Any]) -> ConfluencePage:
|
||||
"""Parst die API-Antwort in ein ConfluencePage-Objekt.
|
||||
|
||||
Args:
|
||||
data: JSON-Response der Confluence API.
|
||||
|
||||
Returns:
|
||||
ConfluencePage mit extrahierten Daten.
|
||||
"""
|
||||
page_id = str(data.get("id", ""))
|
||||
title = data.get("title", "Untitled")
|
||||
|
||||
# Space
|
||||
space = data.get("space", {})
|
||||
space_key = space.get("key", "")
|
||||
|
||||
# Version
|
||||
version_info = data.get("version", {})
|
||||
version = version_info.get("number", 1)
|
||||
last_modified = version_info.get("when", "")
|
||||
|
||||
# Content
|
||||
body = data.get("body", {})
|
||||
storage = body.get("storage", {})
|
||||
content_html = storage.get("value", "")
|
||||
|
||||
# URL
|
||||
base_url = self.base_url
|
||||
links = data.get("_links", {})
|
||||
web_ui = links.get("webui", f"/pages/viewpage.action?pageId={page_id}")
|
||||
url = f"{base_url}{web_ui}"
|
||||
|
||||
# Attachments
|
||||
attachments: list[dict[str, str]] = []
|
||||
children = data.get("children", {})
|
||||
attachment_data = children.get("attachment", {})
|
||||
for att in attachment_data.get("results", []):
|
||||
att_title = att.get("title", "")
|
||||
att_links = att.get("_links", {})
|
||||
att_download = att_links.get("download", "")
|
||||
if att_title:
|
||||
attachments.append({
|
||||
"title": att_title,
|
||||
"url": f"{base_url}{att_download}" if att_download else "",
|
||||
})
|
||||
|
||||
return ConfluencePage(
|
||||
page_id=page_id,
|
||||
title=title,
|
||||
space_key=space_key,
|
||||
version=version,
|
||||
last_modified=last_modified,
|
||||
content_html=content_html,
|
||||
url=url,
|
||||
attachments=attachments,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Confluence Source Strategy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ConfluenceSource(SourceStrategy):
|
||||
"""Quellstrategie: Confluence REST API.
|
||||
|
||||
Unterstützt drei Abrufmodi:
|
||||
1. Einzelne Seiten (params.pages: list[str] mit Page-IDs)
|
||||
2. Seitenbäume (params.tree_roots: list[str] mit Root-Page-IDs)
|
||||
3. Spaces (params.spaces: list[str] mit Space-Keys)
|
||||
|
||||
Inkrementelle Updates werden über die Confluence-Versionsnummer gesteuert.
|
||||
Bekannte Versionen können über set_known_versions() gesetzt werden.
|
||||
|
||||
Fehler bei der API-Kommunikation werden graceful behandelt:
|
||||
- Einzelne fehlgeschlagene Seiten werden übersprungen
|
||||
- Bei kompletten API-Ausfällen wird die gesamte Quelle übersprungen
|
||||
- Bestehende Artefakte werden niemals gelöscht
|
||||
|
||||
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialisiert die ConfluenceSource."""
|
||||
self._known_versions: dict[str, int] = {}
|
||||
|
||||
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
|
||||
"""Extrahiert Artefakte aus Confluence.
|
||||
|
||||
Args:
|
||||
config: SourceConfig mit Confluence-Verbindungsparametern.
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
skipped = 0
|
||||
|
||||
# Client initialisieren
|
||||
try:
|
||||
client = self._create_client(config)
|
||||
except ValueError as e:
|
||||
logger.error("Confluence-Konfigurationsfehler: %s", e)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="auth",
|
||||
message=str(e),
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=False,
|
||||
))
|
||||
return ExtractionResult(artifacts=[], errors=errors, skipped=0)
|
||||
|
||||
# Seiten-IDs sammeln
|
||||
page_ids = self._collect_page_ids(client, config, errors)
|
||||
|
||||
# Seiten abrufen und verarbeiten
|
||||
for page_id in page_ids:
|
||||
try:
|
||||
page = client.get_page(page_id)
|
||||
|
||||
# Inkrementelle Verarbeitung: Version-Check
|
||||
known_version = self._known_versions.get(page_id, 0)
|
||||
if known_version >= page.version:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
artifact = self._page_to_artifact(page, context)
|
||||
artifacts.append(artifact)
|
||||
|
||||
except (urllib.error.URLError, urllib.error.HTTPError) as e:
|
||||
logger.warning(
|
||||
"Seite %s konnte nicht abgerufen werden: %s", page_id, e
|
||||
)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="connection",
|
||||
message=f"Seite {page_id}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=True,
|
||||
artifact_path=f"confluence:{page_id}",
|
||||
))
|
||||
except (json.JSONDecodeError, KeyError, TypeError) as e:
|
||||
logger.warning(
|
||||
"Seite %s konnte nicht geparst werden: %s", page_id, e
|
||||
)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Seite {page_id}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=False,
|
||||
artifact_path=f"confluence:{page_id}",
|
||||
))
|
||||
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors, skipped=skipped)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Confluence unterstützt inkrementelle Verarbeitung via Versions-Nummer.
|
||||
|
||||
Returns:
|
||||
True – inkrementelle Verarbeitung wird unterstützt.
|
||||
"""
|
||||
return True
|
||||
|
||||
def set_known_versions(self, versions: dict[str, int]) -> None:
|
||||
"""Setzt bekannte Seitenversionen für inkrementelle Verarbeitung.
|
||||
|
||||
Args:
|
||||
versions: Mapping von Page-ID zu bekannter Versionsnummer.
|
||||
Seiten mit gleicher oder niedrigerer Version werden übersprungen.
|
||||
"""
|
||||
self._known_versions = dict(versions)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Hilfsmethoden
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _create_client(self, config: SourceConfig) -> ConfluenceClient:
|
||||
"""Erstellt einen ConfluenceClient aus der SourceConfig.
|
||||
|
||||
Args:
|
||||
config: SourceConfig mit Confluence-Parametern.
|
||||
|
||||
Returns:
|
||||
Konfigurierter ConfluenceClient.
|
||||
|
||||
Raises:
|
||||
ValueError: Bei fehlenden Pflichtparametern.
|
||||
"""
|
||||
params = config.params
|
||||
base_url = params.get("base_url", "")
|
||||
if not base_url:
|
||||
raise ValueError("Confluence base_url ist nicht konfiguriert")
|
||||
|
||||
# Token kann unter verschiedenen Keys liegen
|
||||
token = params.get("token", "") or params.get("api_key", "")
|
||||
if not token:
|
||||
raise ValueError(
|
||||
"Confluence Token fehlt (weder 'token' noch 'api_key' in params)"
|
||||
)
|
||||
|
||||
username = params.get("username", "")
|
||||
|
||||
return ConfluenceClient(base_url=base_url, token=token, username=username)
|
||||
|
||||
def _collect_page_ids(
|
||||
self,
|
||||
client: ConfluenceClient,
|
||||
config: SourceConfig,
|
||||
errors: list[SourceError],
|
||||
) -> list[str]:
|
||||
"""Sammelt alle zu verarbeitenden Seiten-IDs.
|
||||
|
||||
Kombiniert IDs aus pages, tree_roots und spaces.
|
||||
|
||||
Args:
|
||||
client: Der Confluence-Client.
|
||||
config: Die Quellkonfiguration.
|
||||
errors: Fehlerliste zum Ergänzen.
|
||||
|
||||
Returns:
|
||||
Deduplizierte Liste von Seiten-IDs.
|
||||
"""
|
||||
params = config.params
|
||||
page_ids: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def _add_ids(ids: list[str]) -> None:
|
||||
for pid in ids:
|
||||
if pid not in seen:
|
||||
seen.add(pid)
|
||||
page_ids.append(pid)
|
||||
|
||||
# 1. Einzelne Seiten
|
||||
pages = params.get("pages", [])
|
||||
if isinstance(pages, list):
|
||||
_add_ids([str(p) for p in pages])
|
||||
|
||||
# 2. Seitenbäume
|
||||
tree_roots = params.get("tree_roots", [])
|
||||
if isinstance(tree_roots, list):
|
||||
for root_id in tree_roots:
|
||||
try:
|
||||
tree_ids = client.get_page_tree(str(root_id))
|
||||
_add_ids(tree_ids)
|
||||
except (urllib.error.URLError, urllib.error.HTTPError) as e:
|
||||
logger.warning(
|
||||
"Seitenbaum %s nicht abrufbar: %s", root_id, e
|
||||
)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="connection",
|
||||
message=f"Seitenbaum {root_id}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=True,
|
||||
))
|
||||
|
||||
# 3. Spaces
|
||||
spaces = params.get("spaces", [])
|
||||
if isinstance(spaces, list):
|
||||
for space_key in spaces:
|
||||
try:
|
||||
space_ids = client.get_space_pages(str(space_key))
|
||||
_add_ids(space_ids)
|
||||
except (urllib.error.URLError, urllib.error.HTTPError) as e:
|
||||
logger.warning(
|
||||
"Space %s nicht abrufbar: %s", space_key, e
|
||||
)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="connection",
|
||||
message=f"Space {space_key}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=True,
|
||||
))
|
||||
|
||||
return page_ids
|
||||
|
||||
def _page_to_artifact(self, page: ConfluencePage, context: str) -> Artifact:
|
||||
"""Konvertiert eine ConfluencePage in ein Artifact.
|
||||
|
||||
Args:
|
||||
page: Die Confluence-Seite mit Inhalt und Metadaten.
|
||||
context: Der Zielkontext.
|
||||
|
||||
Returns:
|
||||
Artifact mit Markdown-Inhalt und angereichertem Frontmatter.
|
||||
"""
|
||||
# HTML → Markdown
|
||||
markdown_content = _html_to_markdown(page.content_html)
|
||||
|
||||
# Titel als Überschrift voranstellen
|
||||
full_content = f"# {page.title}\n\n{markdown_content}"
|
||||
|
||||
# Anhänge als Links anhängen
|
||||
if page.attachments:
|
||||
full_content += "\n\n## Anhänge\n\n"
|
||||
for att in page.attachments:
|
||||
att_title = att.get("title", "Anhang")
|
||||
att_url = att.get("url", "")
|
||||
if att_url:
|
||||
full_content += f"- [{att_title}]({att_url})\n"
|
||||
else:
|
||||
full_content += f"- {att_title}\n"
|
||||
|
||||
# Content-Hash berechnen
|
||||
content_hash = f"sha256:{hashlib.sha256(full_content.encode()).hexdigest()}"
|
||||
|
||||
# Last-Modified parsen
|
||||
created_date = date.today()
|
||||
if page.last_modified:
|
||||
try:
|
||||
# Confluence ISO Format: 2024-01-15T10:30:00.000+01:00
|
||||
dt = datetime.fromisoformat(
|
||||
page.last_modified.replace("Z", "+00:00")
|
||||
)
|
||||
created_date = dt.date()
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Slug für Dateinamen
|
||||
slug = self._slugify(page.title)
|
||||
file_path = Path(f"{context}/knowledge/references/{slug}.md")
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type="reference",
|
||||
title=page.title,
|
||||
tags=[f"space:{page.space_key}"],
|
||||
source_context=context,
|
||||
created=created_date,
|
||||
updated=created_date,
|
||||
shareable=True,
|
||||
content_hash=content_hash,
|
||||
source={
|
||||
"type": "confluence",
|
||||
"url": page.url,
|
||||
"space": page.space_key,
|
||||
"page_id": page.page_id,
|
||||
"version": str(page.version),
|
||||
},
|
||||
category="references",
|
||||
)
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=full_content,
|
||||
file_path=file_path,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _slugify(text: str) -> str:
|
||||
"""Erzeugt einen URL-freundlichen Slug aus einem Text.
|
||||
|
||||
Args:
|
||||
text: Eingabetext (z.B. Seitentitel).
|
||||
|
||||
Returns:
|
||||
Slug in lowercase mit Bindestrichen.
|
||||
"""
|
||||
slug = text.lower().strip()
|
||||
# Umlaute ersetzen
|
||||
replacements = {
|
||||
"ä": "ae", "ö": "oe", "ü": "ue", "ß": "ss",
|
||||
"Ä": "ae", "Ö": "oe", "Ü": "ue",
|
||||
}
|
||||
for char, replacement in replacements.items():
|
||||
slug = slug.replace(char, replacement)
|
||||
# Nicht-alphanumerische Zeichen durch Bindestriche ersetzen
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
||||
# Führende/nachstehende Bindestriche entfernen
|
||||
slug = slug.strip("-")
|
||||
return slug or "untitled"
|
||||
@@ -0,0 +1,433 @@
|
||||
"""Email-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Verarbeitet exportierte E-Mail-Dateien (.eml, .msg) und extrahiert
|
||||
Absender, Empfänger, Betreff, Datum und Body als strukturierte
|
||||
Markdown-Artefakte. Anhänge werden als separate Artefakte referenziert.
|
||||
|
||||
Requirements: 7.1, 7.2, 7.5, 7.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import email
|
||||
import email.utils
|
||||
import hashlib
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from email.message import Message
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["EmailSource"]
|
||||
|
||||
# Supported email file extensions
|
||||
_EMAIL_EXTENSIONS = {".eml", ".msg"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Creates a URL-friendly slug from text."""
|
||||
import re
|
||||
|
||||
slug = text.lower().strip()
|
||||
slug = re.sub(r"[^a-z0-9äöüß\s-]", "", slug)
|
||||
slug = re.sub(r"[\s_]+", "-", slug)
|
||||
slug = re.sub(r"-+", "-", slug)
|
||||
slug = slug.strip("-")
|
||||
return slug[:80] if slug else "untitled"
|
||||
|
||||
|
||||
def _compute_content_hash(content: str) -> str:
|
||||
"""Computes SHA-256 hash of content."""
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
def _parse_email_date(date_str: str | None) -> date | None:
|
||||
"""Parses an email date header into a date object."""
|
||||
if not date_str:
|
||||
return None
|
||||
try:
|
||||
parsed = email.utils.parsedate_to_datetime(date_str)
|
||||
return parsed.date()
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _extract_body(msg: Message) -> str:
|
||||
"""Extracts the body from an email message.
|
||||
|
||||
Prefers plain text, falls back to HTML with basic stripping.
|
||||
"""
|
||||
if msg.is_multipart():
|
||||
plain_parts: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
|
||||
for part in msg.walk():
|
||||
content_type = part.get_content_type()
|
||||
disposition = str(part.get("Content-Disposition", ""))
|
||||
|
||||
# Skip attachments
|
||||
if "attachment" in disposition:
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload is None:
|
||||
continue
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
text = payload.decode(charset, errors="replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
continue
|
||||
|
||||
if content_type == "text/plain":
|
||||
plain_parts.append(text)
|
||||
elif content_type == "text/html":
|
||||
html_parts.append(text)
|
||||
|
||||
if plain_parts:
|
||||
return "\n".join(plain_parts)
|
||||
if html_parts:
|
||||
return _strip_html("\n".join(html_parts))
|
||||
return ""
|
||||
else:
|
||||
# Not multipart
|
||||
try:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload is None:
|
||||
return ""
|
||||
charset = msg.get_content_charset() or "utf-8"
|
||||
text = payload.decode(charset, errors="replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
return ""
|
||||
|
||||
if msg.get_content_type() == "text/html":
|
||||
return _strip_html(text)
|
||||
return text
|
||||
|
||||
|
||||
def _strip_html(html: str) -> str:
|
||||
"""Basic HTML tag stripping for fallback body extraction."""
|
||||
import re
|
||||
|
||||
# Remove script/style blocks
|
||||
text = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
|
||||
# Replace <br> and <p> with newlines
|
||||
text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
|
||||
text = re.sub(r"</?p[^>]*>", "\n", text, flags=re.IGNORECASE)
|
||||
# Remove all remaining tags
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
# Collapse whitespace
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _extract_attachments(msg: Message) -> list[dict[str, str]]:
|
||||
"""Extracts attachment metadata from an email message.
|
||||
|
||||
Returns a list of dicts with 'filename' and 'content_type'.
|
||||
"""
|
||||
attachments: list[dict[str, str]] = []
|
||||
|
||||
if not msg.is_multipart():
|
||||
return attachments
|
||||
|
||||
for part in msg.walk():
|
||||
disposition = str(part.get("Content-Disposition", ""))
|
||||
if "attachment" in disposition:
|
||||
filename = part.get_filename() or "unnamed-attachment"
|
||||
content_type = part.get_content_type() or "application/octet-stream"
|
||||
attachments.append({
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
})
|
||||
|
||||
return attachments
|
||||
|
||||
|
||||
def _format_recipients(recipients_str: str | None) -> list[str]:
|
||||
"""Parses a recipient header into a list of email addresses/names."""
|
||||
if not recipients_str:
|
||||
return []
|
||||
# Parse multiple addresses
|
||||
addresses = email.utils.getaddresses([recipients_str])
|
||||
result: list[str] = []
|
||||
for name, addr in addresses:
|
||||
if name and addr:
|
||||
result.append(f"{name} <{addr}>")
|
||||
elif addr:
|
||||
result.append(addr)
|
||||
elif name:
|
||||
result.append(name)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EmailSource – implementiert SourceStrategy ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EmailSource(SourceStrategy):
|
||||
"""Quellstrategie: Verarbeitet E-Mail-Dateien (.eml/.msg) aus einem Verzeichnis.
|
||||
|
||||
Extrahiert Absender, Empfänger, Betreff, Datum und Body.
|
||||
Erstellt Markdown-Artefakte mit entsprechendem Frontmatter.
|
||||
Anhänge werden als Links im Artefakt-Body referenziert.
|
||||
|
||||
.msg-Dateien erfordern das optionale Paket `extract_msg`. Ist es nicht
|
||||
installiert, wird ein Fehler geloggt und die Datei übersprungen.
|
||||
"""
|
||||
|
||||
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
|
||||
"""Extrahiert E-Mail-Artefakte aus dem konfigurierten Verzeichnis.
|
||||
|
||||
Args:
|
||||
config: Quellkonfiguration mit params["directory"] als Pfad.
|
||||
context: Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und Fehlern.
|
||||
"""
|
||||
directory = self._resolve_directory(config)
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
|
||||
if not directory.exists():
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Verzeichnis existiert nicht: {directory}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
))
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
if not directory.is_dir():
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Pfad ist kein Verzeichnis: {directory}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
))
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
# Find all email files
|
||||
email_files = sorted(
|
||||
f for f in directory.rglob("*")
|
||||
if f.is_file() and f.suffix.lower() in _EMAIL_EXTENSIONS
|
||||
)
|
||||
|
||||
for file_path in email_files:
|
||||
try:
|
||||
artifact = self._process_email_file(file_path, config.name)
|
||||
if artifact is not None:
|
||||
artifacts.append(artifact)
|
||||
except Exception as e:
|
||||
logger.warning("Fehler beim Verarbeiten von %s: %s", file_path, e)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler bei {file_path.name}: {e}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
artifact_path=str(file_path),
|
||||
))
|
||||
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Email-Quellen unterstützen keine inkrementelle Verarbeitung."""
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_directory(self, config: SourceConfig) -> Path:
|
||||
"""Resolves the source directory from config."""
|
||||
dir_param = config.params.get("directory", "")
|
||||
if dir_param:
|
||||
return Path(dir_param)
|
||||
raise ValueError(
|
||||
"EmailSource benötigt params['directory'] in der Konfiguration."
|
||||
)
|
||||
|
||||
def _process_email_file(self, file_path: Path, source_name: str) -> Artifact | None:
|
||||
"""Processes a single email file and returns an Artifact."""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix == ".eml":
|
||||
return self._process_eml(file_path, source_name)
|
||||
elif suffix == ".msg":
|
||||
return self._process_msg(file_path, source_name)
|
||||
return None
|
||||
|
||||
def _process_eml(self, file_path: Path, source_name: str) -> Artifact:
|
||||
"""Processes a .eml file using Python's built-in email module."""
|
||||
raw_bytes = file_path.read_bytes()
|
||||
msg = email.message_from_bytes(raw_bytes)
|
||||
|
||||
# Extract fields
|
||||
sender = msg.get("From", "")
|
||||
to_raw = msg.get("To", "")
|
||||
cc_raw = msg.get("Cc", "")
|
||||
subject = msg.get("Subject", "") or "Kein Betreff"
|
||||
date_str = msg.get("Date", "")
|
||||
message_id = msg.get("Message-ID", "")
|
||||
in_reply_to = msg.get("In-Reply-To", "")
|
||||
|
||||
recipients = _format_recipients(to_raw)
|
||||
cc_recipients = _format_recipients(cc_raw)
|
||||
email_date = _parse_email_date(date_str)
|
||||
|
||||
body = _extract_body(msg)
|
||||
attachments = _extract_attachments(msg)
|
||||
|
||||
# Build artifact
|
||||
return self._build_artifact(
|
||||
file_path=file_path,
|
||||
sender=sender,
|
||||
recipients=recipients,
|
||||
cc_recipients=cc_recipients,
|
||||
subject=subject,
|
||||
email_date=email_date,
|
||||
body=body,
|
||||
attachments=attachments,
|
||||
message_id=message_id,
|
||||
in_reply_to=in_reply_to,
|
||||
source_name=source_name,
|
||||
)
|
||||
|
||||
def _process_msg(self, file_path: Path, source_name: str) -> Artifact:
|
||||
"""Processes a .msg file. Requires extract_msg package."""
|
||||
try:
|
||||
import extract_msg # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
f"Das Paket 'extract_msg' wird für .msg-Dateien benötigt. "
|
||||
f"Installation: pip install extract-msg"
|
||||
)
|
||||
|
||||
msg = extract_msg.Message(str(file_path))
|
||||
try:
|
||||
sender = msg.sender or ""
|
||||
recipients = [r.strip() for r in (msg.to or "").split(";") if r.strip()]
|
||||
cc_recipients = [r.strip() for r in (msg.cc or "").split(";") if r.strip()]
|
||||
subject = msg.subject or "Kein Betreff"
|
||||
body = msg.body or ""
|
||||
email_date = msg.date.date() if msg.date else None
|
||||
message_id = msg.messageId or ""
|
||||
|
||||
attachments = [
|
||||
{"filename": att.longFilename or att.shortFilename or "unnamed",
|
||||
"content_type": att.mimetype or "application/octet-stream"}
|
||||
for att in (msg.attachments or [])
|
||||
]
|
||||
finally:
|
||||
msg.close()
|
||||
|
||||
return self._build_artifact(
|
||||
file_path=file_path,
|
||||
sender=sender,
|
||||
recipients=recipients,
|
||||
cc_recipients=cc_recipients,
|
||||
subject=subject,
|
||||
email_date=email_date,
|
||||
body=body,
|
||||
attachments=attachments,
|
||||
message_id=message_id,
|
||||
in_reply_to="",
|
||||
source_name=source_name,
|
||||
)
|
||||
|
||||
def _build_artifact(
|
||||
self,
|
||||
file_path: Path,
|
||||
sender: str,
|
||||
recipients: list[str],
|
||||
cc_recipients: list[str],
|
||||
subject: str,
|
||||
email_date: date | None,
|
||||
body: str,
|
||||
attachments: list[dict[str, str]],
|
||||
message_id: str,
|
||||
in_reply_to: str,
|
||||
source_name: str,
|
||||
) -> Artifact:
|
||||
"""Builds a Markdown artifact from parsed email data."""
|
||||
# Build Markdown content
|
||||
content_lines: list[str] = []
|
||||
content_lines.append(f"# {subject}\n")
|
||||
content_lines.append("")
|
||||
content_lines.append(f"**Von:** {sender} ")
|
||||
content_lines.append(f"**An:** {', '.join(recipients)} ")
|
||||
if cc_recipients:
|
||||
content_lines.append(f"**CC:** {', '.join(cc_recipients)} ")
|
||||
if email_date:
|
||||
content_lines.append(f"**Datum:** {email_date.isoformat()} ")
|
||||
content_lines.append("")
|
||||
content_lines.append("---\n")
|
||||
content_lines.append("")
|
||||
content_lines.append(body.strip())
|
||||
|
||||
# Attachments section
|
||||
if attachments:
|
||||
content_lines.append("")
|
||||
content_lines.append("")
|
||||
content_lines.append("## Anhänge\n")
|
||||
content_lines.append("")
|
||||
for att in attachments:
|
||||
filename = att["filename"]
|
||||
content_lines.append(f"- [{filename}](attachments/{filename})")
|
||||
|
||||
content = "\n".join(content_lines)
|
||||
|
||||
# Build source dict for frontmatter
|
||||
source_dict: dict[str, str] = {
|
||||
"type": "email",
|
||||
"file": str(file_path),
|
||||
}
|
||||
if message_id:
|
||||
source_dict["message_id"] = message_id
|
||||
|
||||
# Build metadata
|
||||
metadata = ArtifactMetadata(
|
||||
type="email",
|
||||
title=subject,
|
||||
tags=["email"],
|
||||
source_context="",
|
||||
created=email_date,
|
||||
updated=email_date,
|
||||
shareable=False,
|
||||
source=source_dict,
|
||||
category="inbox",
|
||||
people=[sender] + recipients,
|
||||
content_hash=_compute_content_hash(content),
|
||||
)
|
||||
|
||||
# Generate a filename from subject
|
||||
date_prefix = email_date.isoformat() if email_date else "undated"
|
||||
slug = _slugify(subject)
|
||||
filename = f"{date_prefix}-{slug}.md"
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=Path(filename),
|
||||
)
|
||||
@@ -0,0 +1,483 @@
|
||||
"""Jira-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Extrahiert Issues via JQL-Filter aus der Jira REST API und liefert
|
||||
sie als strukturierte Markdown-Artefakte zurück.
|
||||
|
||||
Unterstützt inkrementelle Updates via das `updated`-Feld der Jira-Issues.
|
||||
|
||||
Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["JiraSource"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hilfsfunktionen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_content_hash(text: str) -> str:
|
||||
"""Berechnet SHA-256-Hash eines Textes."""
|
||||
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
def _parse_jira_date(date_str: str | None) -> date | None:
|
||||
"""Parsed ein Jira-Datumsfeld (ISO-8601) in ein date-Objekt.
|
||||
|
||||
Jira liefert Timestamps wie '2025-07-01T10:30:00.000+0200'.
|
||||
"""
|
||||
if not date_str:
|
||||
return None
|
||||
try:
|
||||
# Versuche volles Datetime-Format
|
||||
dt = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
|
||||
return dt.date()
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
try:
|
||||
return date.fromisoformat(date_str[:10])
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Erzeugt einen URL-freundlichen Slug aus einem Text."""
|
||||
import re
|
||||
|
||||
slug = text.lower().strip()
|
||||
slug = re.sub(r"[^a-z0-9\-]+", "-", slug)
|
||||
slug = re.sub(r"-+", "-", slug)
|
||||
return slug.strip("-")
|
||||
|
||||
|
||||
def _build_issue_markdown(issue: dict[str, Any]) -> str:
|
||||
"""Stellt ein Jira-Issue als Markdown-Body zusammen.
|
||||
|
||||
Enthält: Summary als H1, Description, Kommentare als Sub-Sections.
|
||||
"""
|
||||
fields = issue.get("fields", {})
|
||||
summary = fields.get("summary", "Untitled Issue")
|
||||
description = fields.get("description") or ""
|
||||
status = fields.get("status", {}).get("name", "Unknown")
|
||||
|
||||
parts: list[str] = []
|
||||
|
||||
# Title
|
||||
parts.append(f"# {summary}\n")
|
||||
|
||||
# Status-Info
|
||||
parts.append(f"**Status:** {status}\n")
|
||||
|
||||
# Description
|
||||
if description:
|
||||
parts.append("## Description\n")
|
||||
parts.append(f"{description}\n")
|
||||
|
||||
# Kommentare
|
||||
comment_data = fields.get("comment", {})
|
||||
comments = comment_data.get("comments", []) if isinstance(comment_data, dict) else []
|
||||
|
||||
if comments:
|
||||
parts.append("## Comments\n")
|
||||
for comment in comments:
|
||||
author = comment.get("author", {}).get("displayName", "Unknown")
|
||||
created = comment.get("created", "")[:10]
|
||||
body = comment.get("body", "")
|
||||
parts.append(f"### {author} ({created})\n")
|
||||
parts.append(f"{body}\n")
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _build_frontmatter(issue: dict[str, Any], base_url: str) -> dict[str, str]:
|
||||
"""Erstellt die Frontmatter-Source-Felder für ein Jira-Issue."""
|
||||
fields = issue.get("fields", {})
|
||||
key = issue.get("key", "")
|
||||
project = fields.get("project", {}).get("key", "")
|
||||
status = fields.get("status", {}).get("name", "")
|
||||
assignee = fields.get("assignee", {})
|
||||
reporter = fields.get("reporter", {})
|
||||
|
||||
assignee_name = assignee.get("displayName", "") if assignee else ""
|
||||
reporter_name = reporter.get("displayName", "") if reporter else ""
|
||||
|
||||
# Construct browse URL
|
||||
jira_url = f"{base_url.rstrip('/')}/browse/{key}" if base_url else ""
|
||||
|
||||
return {
|
||||
"type": "jira",
|
||||
"issue_key": key,
|
||||
"project": project,
|
||||
"status": status,
|
||||
"assignee": assignee_name,
|
||||
"reporter": reporter_name,
|
||||
"url": jira_url,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JiraSource – implementiert SourceStrategy ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class JiraSource(SourceStrategy):
|
||||
"""Quellstrategie: Jira-Issues via JQL-Filter extrahieren.
|
||||
|
||||
Verwendet die Jira REST API (v2) um Issues basierend auf einem
|
||||
konfigurierbaren JQL-Filter abzurufen und als Markdown-Artefakte
|
||||
zurückzugeben.
|
||||
|
||||
Unterstützt inkrementelle Verarbeitung: Beim wiederholten Aufruf
|
||||
können über `_last_updated` bereits bekannte Update-Zeitstempel
|
||||
übergeben werden, um nur geänderte Issues zu verarbeiten.
|
||||
|
||||
SourceConfig params:
|
||||
base_url: Jira-Instanz-URL (z.B. "https://jira.bahn.de")
|
||||
api_key / token: API-Token für Authentifizierung
|
||||
username: Benutzername für Basic Auth (optional)
|
||||
jql: JQL-Filter-String
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialisiert die JiraSource."""
|
||||
self._last_updated: dict[str, str] = {}
|
||||
self._session: requests.Session | None = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SourceStrategy ABC Interface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
|
||||
"""Extrahiert Jira-Issues basierend auf der Quellkonfiguration.
|
||||
|
||||
Args:
|
||||
config: SourceConfig mit Jira-Verbindungsparametern.
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
params = config.params
|
||||
base_url = params.get("base_url", "")
|
||||
token = params.get("api_key") or params.get("token", "")
|
||||
username = params.get("username", "")
|
||||
jql = params.get("jql", "")
|
||||
|
||||
if not base_url:
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="config",
|
||||
message="Missing required param 'base_url'",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
if not jql:
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="config",
|
||||
message="Missing required param 'jql'",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
# Session aufbauen
|
||||
session = self._create_session(token, username)
|
||||
|
||||
# Issues abrufen
|
||||
try:
|
||||
issues = self._fetch_issues(session, base_url, jql)
|
||||
except JiraAPIError as e:
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type=e.error_type,
|
||||
message=str(e),
|
||||
timestamp="",
|
||||
retry=e.retry,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
# Issues zu Artefakten konvertieren
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
skipped = 0
|
||||
|
||||
for issue in issues:
|
||||
key = issue.get("key", "unknown")
|
||||
fields = issue.get("fields", {})
|
||||
updated_str = fields.get("updated", "")
|
||||
|
||||
# Inkrementelle Verarbeitung: Skip wenn nicht geändert
|
||||
if self._last_updated.get(key) == updated_str and updated_str:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
artifact = self._issue_to_artifact(issue, base_url, context)
|
||||
artifacts.append(artifact)
|
||||
# Update-Timestamp merken
|
||||
if updated_str:
|
||||
self._last_updated[key] = updated_str
|
||||
except Exception as e:
|
||||
logger.warning("Fehler beim Verarbeiten von %s: %s", key, e)
|
||||
errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Error processing issue {key}: {e}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
artifact_path=key,
|
||||
)
|
||||
)
|
||||
|
||||
return ExtractionResult(
|
||||
artifacts=artifacts,
|
||||
errors=errors,
|
||||
skipped=skipped,
|
||||
)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Jira unterstützt inkrementelle Verarbeitung via updated-Feld.
|
||||
|
||||
Returns:
|
||||
True – inkrementelle Verarbeitung wird unterstützt.
|
||||
"""
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inkrementelle Verarbeitung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def set_last_updated(self, timestamps: dict[str, str]) -> None:
|
||||
"""Setzt die bekannten Update-Timestamps für inkrementelle Verarbeitung.
|
||||
|
||||
Args:
|
||||
timestamps: Mapping von Issue-Key zu letztem `updated`-Timestamp.
|
||||
"""
|
||||
self._last_updated = dict(timestamps)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Implementierung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _create_session(self, token: str, username: str) -> requests.Session:
|
||||
"""Erstellt eine konfigurierte HTTP-Session für Jira.
|
||||
|
||||
Args:
|
||||
token: API-Token oder Passwort.
|
||||
username: Benutzername (für Basic Auth).
|
||||
|
||||
Returns:
|
||||
Konfigurierte requests.Session.
|
||||
"""
|
||||
session = requests.Session()
|
||||
session.headers["Accept"] = "application/json"
|
||||
session.headers["Content-Type"] = "application/json"
|
||||
|
||||
if username and token:
|
||||
# Basic Auth (Jira Server/DC)
|
||||
session.auth = (username, token)
|
||||
elif token:
|
||||
# Bearer Token (Jira Cloud)
|
||||
session.headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
return session
|
||||
|
||||
def _fetch_issues(
|
||||
self,
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
jql: str,
|
||||
max_results: int = 100,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Ruft Issues aus der Jira REST API ab.
|
||||
|
||||
Paginiert automatisch bei großen Ergebnismengen.
|
||||
|
||||
Args:
|
||||
session: Die HTTP-Session.
|
||||
base_url: Jira-Instanz-URL.
|
||||
jql: JQL-Filter.
|
||||
max_results: Maximale Anzahl pro Request.
|
||||
|
||||
Returns:
|
||||
Liste von Issue-Dictionaries.
|
||||
|
||||
Raises:
|
||||
JiraAPIError: Bei API-Fehlern.
|
||||
"""
|
||||
url = f"{base_url.rstrip('/')}/rest/api/2/search"
|
||||
all_issues: list[dict[str, Any]] = []
|
||||
start_at = 0
|
||||
|
||||
while True:
|
||||
params = {
|
||||
"jql": jql,
|
||||
"startAt": start_at,
|
||||
"maxResults": max_results,
|
||||
"fields": "summary,description,status,assignee,reporter,"
|
||||
"project,comment,updated,created",
|
||||
}
|
||||
|
||||
try:
|
||||
response = session.get(url, params=params, timeout=30)
|
||||
except requests.ConnectionError as e:
|
||||
raise JiraAPIError(
|
||||
f"Connection failed to {base_url}: {e}",
|
||||
error_type="connection",
|
||||
retry=True,
|
||||
)
|
||||
except requests.Timeout as e:
|
||||
raise JiraAPIError(
|
||||
f"Request timeout to {base_url}: {e}",
|
||||
error_type="connection",
|
||||
retry=True,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
raise JiraAPIError(
|
||||
f"Request error to {base_url}: {e}",
|
||||
error_type="connection",
|
||||
retry=True,
|
||||
)
|
||||
|
||||
if response.status_code == 401:
|
||||
raise JiraAPIError(
|
||||
f"Authentication failed for {base_url}",
|
||||
error_type="auth",
|
||||
retry=False,
|
||||
)
|
||||
elif response.status_code == 403:
|
||||
raise JiraAPIError(
|
||||
f"Access forbidden for {base_url}",
|
||||
error_type="auth",
|
||||
retry=False,
|
||||
)
|
||||
elif response.status_code >= 400:
|
||||
raise JiraAPIError(
|
||||
f"API error {response.status_code} from {base_url}: "
|
||||
f"{response.text[:200]}",
|
||||
error_type="connection",
|
||||
retry=response.status_code >= 500,
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
issues = data.get("issues", [])
|
||||
all_issues.extend(issues)
|
||||
|
||||
# Pagination
|
||||
total = data.get("total", 0)
|
||||
start_at += len(issues)
|
||||
if start_at >= total or not issues:
|
||||
break
|
||||
|
||||
return all_issues
|
||||
|
||||
def _issue_to_artifact(
|
||||
self, issue: dict[str, Any], base_url: str, context: str
|
||||
) -> Artifact:
|
||||
"""Konvertiert ein Jira-Issue-Dict in ein Artifact.
|
||||
|
||||
Args:
|
||||
issue: Das Issue-Dictionary von der Jira API.
|
||||
base_url: Jira-Instanz-URL für Link-Generierung.
|
||||
context: Der Zielkontext.
|
||||
|
||||
Returns:
|
||||
Ein Artifact mit Markdown-Inhalt und angereichertem Frontmatter.
|
||||
"""
|
||||
fields = issue.get("fields", {})
|
||||
key = issue.get("key", "unknown")
|
||||
summary = fields.get("summary", "Untitled")
|
||||
|
||||
# Markdown-Body zusammenstellen
|
||||
content = _build_issue_markdown(issue)
|
||||
|
||||
# Content-Hash
|
||||
content_hash = _compute_content_hash(content)
|
||||
|
||||
# Source-Frontmatter
|
||||
source_dict = _build_frontmatter(issue, base_url)
|
||||
|
||||
# Daten parsen
|
||||
created = _parse_jira_date(fields.get("created"))
|
||||
updated = _parse_jira_date(fields.get("updated"))
|
||||
|
||||
# Tags aus Projekt und Status ableiten
|
||||
project_key = fields.get("project", {}).get("key", "")
|
||||
status_name = fields.get("status", {}).get("name", "")
|
||||
tags: list[str] = []
|
||||
if project_key:
|
||||
tags.append(project_key.lower())
|
||||
if status_name:
|
||||
tags.append(_slugify(status_name))
|
||||
|
||||
# Metadata zusammenbauen
|
||||
metadata = ArtifactMetadata(
|
||||
type="jira",
|
||||
title=summary,
|
||||
tags=tags,
|
||||
source_context=context,
|
||||
created=created,
|
||||
updated=updated,
|
||||
shareable=False,
|
||||
content_hash=content_hash,
|
||||
source=source_dict,
|
||||
category="project",
|
||||
)
|
||||
|
||||
# Dateiname aus Issue-Key ableiten
|
||||
from pathlib import Path
|
||||
|
||||
filename = f"{key.lower()}.md"
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=Path(filename),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fehlerklasse
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class JiraAPIError(Exception):
|
||||
"""Fehler bei der Kommunikation mit der Jira REST API."""
|
||||
|
||||
def __init__(self, message: str, error_type: str = "connection", retry: bool = True):
|
||||
super().__init__(message)
|
||||
self.error_type = error_type
|
||||
self.retry = retry
|
||||
@@ -0,0 +1,426 @@
|
||||
"""Link-Source und LinkRegistry für Web-Link-Management.
|
||||
|
||||
Verwaltet Web-Links als Wissensartefakte vom Typ `link` im `links/`-Unterordner
|
||||
des jeweiligen Kontext-Knowledge-Folders. Unterstützt URL-Deduplication mit
|
||||
Tag-Merging, Metadaten-Fetch und Suche über URL, Titel und Tags.
|
||||
|
||||
Requirements: 8.1, 8.2, 8.4, 8.5, 8.6, 8.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
Artifact,
|
||||
ArtifactMetadata,
|
||||
parse_frontmatter,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LinkRegistry:
|
||||
"""Manages web links as knowledge artifacts.
|
||||
|
||||
Links are stored as Markdown files with YAML-Frontmatter in
|
||||
`{context_path}/links/`. Each link artifact has type: link and contains
|
||||
url, title, tags, created in its frontmatter.
|
||||
|
||||
Supports:
|
||||
- URL deduplication with tag merging (Req 8.7)
|
||||
- Metadata fetching from target URL (Req 8.4)
|
||||
- Search over URL, title, and tags (Req 8.6)
|
||||
- fetch_failed flag when URL is unreachable (Req 8.5)
|
||||
"""
|
||||
|
||||
def __init__(self, context_path: Path) -> None:
|
||||
"""Initialize LinkRegistry with a context knowledge path.
|
||||
|
||||
Args:
|
||||
context_path: Path to the context's knowledge folder,
|
||||
e.g. `{monorepo_root}/bahn/knowledge/`.
|
||||
"""
|
||||
self.context_path = context_path
|
||||
self.links_dir = context_path / "links"
|
||||
|
||||
def save_link(
|
||||
self,
|
||||
url: str,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
) -> Path:
|
||||
"""Save or update a link artifact. Deduplicates by URL.
|
||||
|
||||
If a link with the same URL already exists, merges tags (union of
|
||||
existing + new tags) instead of creating a duplicate.
|
||||
|
||||
Args:
|
||||
url: The web URL to save.
|
||||
title: Optional title override. If None, fetched from URL.
|
||||
tags: Optional list of tags.
|
||||
|
||||
Returns:
|
||||
Path of the created or updated link artifact.
|
||||
"""
|
||||
if tags is None:
|
||||
tags = []
|
||||
|
||||
# Check for existing link with same URL
|
||||
existing_path = self._find_existing(url)
|
||||
|
||||
if existing_path is not None:
|
||||
return self._merge_tags(existing_path, tags)
|
||||
|
||||
# Fetch metadata if title not provided
|
||||
fetched_title = ""
|
||||
fetched_description = ""
|
||||
fetch_failed = False
|
||||
|
||||
if title is None:
|
||||
fetched_title, fetched_description = self._fetch_metadata(url)
|
||||
if fetched_title:
|
||||
title = fetched_title
|
||||
else:
|
||||
# Use domain + path as fallback title
|
||||
title = self._title_from_url(url)
|
||||
fetch_failed = True
|
||||
else:
|
||||
# Still try to fetch for description, but don't override title
|
||||
try:
|
||||
_, fetched_description = self._fetch_metadata(url)
|
||||
except Exception:
|
||||
fetch_failed = True
|
||||
|
||||
# Generate filename from title
|
||||
slug = self._slugify(title)
|
||||
if not slug:
|
||||
slug = self._slugify(self._title_from_url(url))
|
||||
if not slug:
|
||||
slug = "untitled-link"
|
||||
filename = f"{slug}.md"
|
||||
|
||||
# Ensure links directory exists
|
||||
self.links_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Avoid filename collisions
|
||||
target_path = self.links_dir / filename
|
||||
counter = 1
|
||||
while target_path.exists():
|
||||
target_path = self.links_dir / f"{slug}-{counter}.md"
|
||||
counter += 1
|
||||
|
||||
# Build the artifact content
|
||||
content = self._build_link_content(
|
||||
url=url,
|
||||
title=title,
|
||||
tags=tags,
|
||||
description=fetched_description,
|
||||
fetch_failed=fetch_failed,
|
||||
)
|
||||
|
||||
target_path.write_text(content, encoding="utf-8")
|
||||
return target_path
|
||||
|
||||
def search_links(self, query: str) -> list[Artifact]:
|
||||
"""Search links by URL, title, or tags.
|
||||
|
||||
Performs case-insensitive substring matching against the URL,
|
||||
title, and all tags of each link artifact.
|
||||
|
||||
Args:
|
||||
query: Search string to match against link metadata.
|
||||
|
||||
Returns:
|
||||
List of matching Artifact objects.
|
||||
"""
|
||||
if not self.links_dir.exists():
|
||||
return []
|
||||
|
||||
query_lower = query.lower()
|
||||
results: list[Artifact] = []
|
||||
|
||||
for md_file in sorted(self.links_dir.glob("*.md")):
|
||||
try:
|
||||
artifact = parse_frontmatter(md_file)
|
||||
except (ValueError, OSError) as e:
|
||||
logger.warning("Skipping unparseable link file %s: %s", md_file, e)
|
||||
continue
|
||||
|
||||
# Only consider link-type artifacts
|
||||
if artifact.metadata.type != "link":
|
||||
continue
|
||||
|
||||
# Match against URL
|
||||
url = artifact.metadata.source.get("url", "")
|
||||
if query_lower in url.lower():
|
||||
results.append(artifact)
|
||||
continue
|
||||
|
||||
# Match against title
|
||||
if query_lower in artifact.metadata.title.lower():
|
||||
results.append(artifact)
|
||||
continue
|
||||
|
||||
# Match against tags
|
||||
if any(query_lower in tag.lower() for tag in artifact.metadata.tags):
|
||||
results.append(artifact)
|
||||
continue
|
||||
|
||||
return results
|
||||
|
||||
def _fetch_metadata(self, url: str) -> tuple[str, str]:
|
||||
"""Fetch page title and description from URL.
|
||||
|
||||
Makes an HTTP GET request to the URL and tries to extract
|
||||
the <title> tag and meta description.
|
||||
|
||||
Args:
|
||||
url: The target URL to fetch.
|
||||
|
||||
Returns:
|
||||
Tuple of (title, description). Empty strings if fetch fails.
|
||||
"""
|
||||
try:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": "MonorepoCLI/1.0 LinkRegistry"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
# Read limited content (first 64KB) to avoid huge downloads
|
||||
raw = response.read(65536)
|
||||
charset = response.headers.get_content_charset() or "utf-8"
|
||||
try:
|
||||
html = raw.decode(charset, errors="replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
html = raw.decode("utf-8", errors="replace")
|
||||
|
||||
title = self._extract_html_title(html)
|
||||
description = self._extract_meta_description(html)
|
||||
return title, description
|
||||
|
||||
except Exception as e:
|
||||
logger.debug("Failed to fetch metadata from %s: %s", url, e)
|
||||
return "", ""
|
||||
|
||||
def _find_existing(self, url: str) -> Path | None:
|
||||
"""Check for duplicate URL in context links folder.
|
||||
|
||||
Scans all Markdown files in the links directory and checks
|
||||
if any has a matching URL in its frontmatter source.url field.
|
||||
|
||||
Args:
|
||||
url: The URL to check for duplicates.
|
||||
|
||||
Returns:
|
||||
Path to the existing artifact, or None if not found.
|
||||
"""
|
||||
if not self.links_dir.exists():
|
||||
return None
|
||||
|
||||
for md_file in self.links_dir.glob("*.md"):
|
||||
try:
|
||||
artifact = parse_frontmatter(md_file)
|
||||
except (ValueError, OSError):
|
||||
continue
|
||||
|
||||
if artifact.metadata.type != "link":
|
||||
continue
|
||||
|
||||
existing_url = artifact.metadata.source.get("url", "")
|
||||
if existing_url == url:
|
||||
return md_file
|
||||
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _merge_tags(self, existing_path: Path, new_tags: list[str]) -> Path:
|
||||
"""Merge new tags into an existing link artifact.
|
||||
|
||||
Reads the existing artifact, computes union of tags,
|
||||
and rewrites the file if tags changed.
|
||||
|
||||
Args:
|
||||
existing_path: Path to the existing link artifact.
|
||||
new_tags: New tags to merge in.
|
||||
|
||||
Returns:
|
||||
Path to the updated artifact.
|
||||
"""
|
||||
artifact = parse_frontmatter(existing_path)
|
||||
|
||||
# Compute tag union (preserving order, existing first)
|
||||
existing_tags = list(artifact.metadata.tags)
|
||||
merged = list(existing_tags)
|
||||
for tag in new_tags:
|
||||
if tag not in merged:
|
||||
merged.append(tag)
|
||||
|
||||
if merged == existing_tags:
|
||||
# No change needed
|
||||
return existing_path
|
||||
|
||||
# Rewrite with updated tags
|
||||
artifact.metadata.tags = merged
|
||||
content = self._render_artifact(artifact)
|
||||
existing_path.write_text(content, encoding="utf-8")
|
||||
return existing_path
|
||||
|
||||
def _render_artifact(self, artifact: Artifact) -> str:
|
||||
"""Render an artifact back to its file content (frontmatter + body).
|
||||
|
||||
Args:
|
||||
artifact: The artifact to render.
|
||||
|
||||
Returns:
|
||||
Complete file content as string.
|
||||
"""
|
||||
from monorepo.knowledge.artifact import _generate_frontmatter
|
||||
|
||||
frontmatter = _generate_frontmatter(artifact.metadata)
|
||||
return frontmatter + artifact.content
|
||||
|
||||
def _build_link_content(
|
||||
self,
|
||||
url: str,
|
||||
title: str,
|
||||
tags: list[str],
|
||||
description: str,
|
||||
fetch_failed: bool,
|
||||
) -> str:
|
||||
"""Build the complete Markdown content for a link artifact.
|
||||
|
||||
Args:
|
||||
url: The link URL.
|
||||
title: The link title.
|
||||
tags: List of tags.
|
||||
description: Fetched description (may be empty).
|
||||
fetch_failed: Whether metadata fetch failed.
|
||||
|
||||
Returns:
|
||||
Complete Markdown file content with frontmatter.
|
||||
"""
|
||||
today = date.today().isoformat()
|
||||
|
||||
# Build frontmatter data
|
||||
data: dict = {
|
||||
"type": "link",
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
"created": today,
|
||||
"source": {"type": "link", "url": url},
|
||||
"source_context": "",
|
||||
"shareable": False,
|
||||
}
|
||||
|
||||
if fetch_failed:
|
||||
data["fetch_failed"] = True
|
||||
|
||||
# Serialize frontmatter
|
||||
yaml_str = yaml.dump(
|
||||
data,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
frontmatter = f"---\n{yaml_str}---\n"
|
||||
|
||||
# Build body
|
||||
body_lines = [f"\n[{title}]({url})\n"]
|
||||
if description:
|
||||
body_lines.append(f"\n{description}\n")
|
||||
|
||||
body = "".join(body_lines)
|
||||
return frontmatter + body
|
||||
|
||||
def _title_from_url(self, url: str) -> str:
|
||||
"""Derive a title from a URL (domain + path).
|
||||
|
||||
Args:
|
||||
url: The URL to derive title from.
|
||||
|
||||
Returns:
|
||||
A human-readable title string.
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
path_part = parsed.path.strip("/")
|
||||
if path_part:
|
||||
return f"{parsed.netloc}/{path_part}"
|
||||
return parsed.netloc or url
|
||||
|
||||
def _slugify(self, text: str) -> str:
|
||||
"""Convert text to URL-safe slug.
|
||||
|
||||
Lowercase, replace non-alphanumeric with hyphens,
|
||||
collapse multiple hyphens, limit to 60 chars.
|
||||
|
||||
Args:
|
||||
text: The text to slugify.
|
||||
|
||||
Returns:
|
||||
A URL-safe slug string.
|
||||
"""
|
||||
slug = text.lower()
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
||||
slug = re.sub(r"-{2,}", "-", slug)
|
||||
slug = slug.strip("-")
|
||||
return slug[:60]
|
||||
|
||||
def _extract_html_title(self, html: str) -> str:
|
||||
"""Extract <title> content from HTML.
|
||||
|
||||
Args:
|
||||
html: The HTML content string.
|
||||
|
||||
Returns:
|
||||
The title text, or empty string if not found.
|
||||
"""
|
||||
match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# Clean up HTML entities
|
||||
title = re.sub(r"&", "&", title)
|
||||
title = re.sub(r"<", "<", title)
|
||||
title = re.sub(r">", ">", title)
|
||||
title = re.sub(r""", '"', title)
|
||||
title = re.sub(r"'", "'", title)
|
||||
title = re.sub(r"\s+", " ", title)
|
||||
return title
|
||||
return ""
|
||||
|
||||
def _extract_meta_description(self, html: str) -> str:
|
||||
"""Extract meta description from HTML.
|
||||
|
||||
Args:
|
||||
html: The HTML content string.
|
||||
|
||||
Returns:
|
||||
The description text, or empty string if not found.
|
||||
"""
|
||||
match = re.search(
|
||||
r'<meta\s+[^>]*name=["\']description["\'][^>]*content=["\']([^"\']*)["\']',
|
||||
html,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if not match:
|
||||
# Try alternative ordering (content before name)
|
||||
match = re.search(
|
||||
r'<meta\s+[^>]*content=["\']([^"\']*)["\'][^>]*name=["\']description["\']',
|
||||
html,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ""
|
||||
@@ -1,111 +1,376 @@
|
||||
"""Markdown-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Scannt ein Verzeichnis rekursiv nach .md-Dateien mit YAML-Frontmatter
|
||||
Scannt ein Verzeichnis rekursiv nach .md/.txt-Dateien mit YAML-Frontmatter
|
||||
und liefert eine Liste von Artifacts zurück.
|
||||
|
||||
Requirements: 3.2, 3.10, 3.12, 3.19
|
||||
Implementiert das SourceStrategy-Interface (ABC) und unterstützt
|
||||
inkrementelle Verarbeitung via Content-Hash.
|
||||
|
||||
Requirements: 2.2, 2.4, 2.7, 4.1
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, parse_frontmatter
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata, parse_frontmatter
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError as BaseSourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legacy-kompatibles SourceError (behält file_path/error-Interface)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceError:
|
||||
"""Fehler bei der Verarbeitung einer einzelnen Quelldatei."""
|
||||
"""Fehler bei der Verarbeitung einer einzelnen Quelldatei.
|
||||
|
||||
Behält das alte Interface (file_path, error, retry) für Abwärtskompatibilität
|
||||
mit ETLPipeline und bestehenden Tests.
|
||||
"""
|
||||
|
||||
file_path: str
|
||||
error: str
|
||||
retry: bool = True
|
||||
|
||||
def to_base_error(self, source_name: str = "markdown") -> BaseSourceError:
|
||||
"""Konvertiert in das neue BaseSourceError-Format."""
|
||||
return BaseSourceError(
|
||||
source_name=source_name,
|
||||
error_type="parse",
|
||||
message=self.error,
|
||||
timestamp="",
|
||||
retry=self.retry,
|
||||
artifact_path=self.file_path,
|
||||
)
|
||||
|
||||
@dataclass
|
||||
class MarkdownSource:
|
||||
"""Quellstrategie: Rekursives Scannen eines Verzeichnisses nach Markdown-Dateien.
|
||||
|
||||
Findet alle .md-Dateien im angegebenen Verzeichnis (rekursiv),
|
||||
__all__ = ["MarkdownSource", "SourceError"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content-Hash-Hilfsfunktion für inkrementelle Verarbeitung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_file_hash(file_path: Path) -> str:
|
||||
"""Berechnet einen SHA-256-Hash des Dateiinhalts.
|
||||
|
||||
Wird für inkrementelle Verarbeitung verwendet – nur geänderte
|
||||
Dateien werden erneut verarbeitet.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
|
||||
Returns:
|
||||
Hash-String im Format "sha256:<hex>".
|
||||
"""
|
||||
content = file_path.read_bytes()
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MarkdownSource – implementiert SourceStrategy ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Unterstützte Dateiendungen
|
||||
_SUPPORTED_EXTENSIONS = {".md", ".txt"}
|
||||
|
||||
|
||||
class MarkdownSource(SourceStrategy):
|
||||
"""Quellstrategie: Rekursives Scannen eines Verzeichnisses nach Markdown/Text-Dateien.
|
||||
|
||||
Findet alle .md- und .txt-Dateien im angegebenen Verzeichnis (rekursiv),
|
||||
parsed jede Datei mit parse_frontmatter und liefert die resultierenden
|
||||
Artifacts. Dateien ohne gültiges YAML-Frontmatter werden übersprungen
|
||||
und als Warning geloggt.
|
||||
Artifacts. Dateien ohne gültiges YAML-Frontmatter werden als Plain-Text-
|
||||
Artefakte mit minimalem Frontmatter behandelt (.txt) oder als Fehler
|
||||
geloggt (.md).
|
||||
|
||||
Implementiert das SourceStrategy-Interface und unterstützt zusätzlich
|
||||
die alte Convenience-Schnittstelle `MarkdownSource(directory=...)`.
|
||||
|
||||
Inkrementelle Verarbeitung:
|
||||
Bei wiederholtem Aufruf können über `_known_hashes` bereits bekannte
|
||||
Datei-Hashes übergeben werden. Dateien mit unverändertem Hash werden
|
||||
übersprungen (gezählt in `ExtractionResult.skipped`).
|
||||
"""
|
||||
|
||||
directory: Path
|
||||
errors: list[SourceError] = field(default_factory=list)
|
||||
def __init__(self, directory: Path | None = None) -> None:
|
||||
"""Initialisiert die MarkdownSource.
|
||||
|
||||
def extract(self) -> list[Artifact]:
|
||||
"""Extrahiert alle Artefakte aus dem Quellverzeichnis.
|
||||
Args:
|
||||
directory: Optionaler Pfad zum Quellverzeichnis (Convenience-Interface).
|
||||
Wenn angegeben, kann extract() ohne config aufgerufen werden.
|
||||
"""
|
||||
self._directory: Path | None = directory
|
||||
self.errors: list[SourceError] = []
|
||||
self._known_hashes: dict[str, str] = {}
|
||||
|
||||
Durchsucht das Verzeichnis rekursiv nach .md-Dateien,
|
||||
parsed jede Datei und sammelt Fehler ohne abzubrechen.
|
||||
# ------------------------------------------------------------------
|
||||
# SourceStrategy ABC Interface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def extract( # type: ignore[override]
|
||||
self,
|
||||
config: SourceConfig | None = None,
|
||||
context: str = "",
|
||||
) -> ExtractionResult | list[Artifact]:
|
||||
"""Extrahiert Artefakte aus dem konfigurierten Verzeichnis.
|
||||
|
||||
Unterstützt zwei Aufrufmodi:
|
||||
1. Neues Interface: extract(config, context) → ExtractionResult
|
||||
2. Legacy-Interface: extract() → list[Artifact]
|
||||
|
||||
Args:
|
||||
config: Die Quellkonfiguration mit Verzeichnis in params["directory"].
|
||||
Wenn None, wird self._directory verwendet (Legacy-Modus).
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
Liste erfolgreich geparster Artifacts.
|
||||
ExtractionResult (neues Interface) oder list[Artifact] (Legacy).
|
||||
"""
|
||||
self.errors = []
|
||||
artifacts: list[Artifact] = []
|
||||
# Bestimme ob neues oder altes Interface verwendet wird
|
||||
use_legacy = config is None
|
||||
|
||||
if not self.directory.exists():
|
||||
logger.warning(
|
||||
"Quellverzeichnis existiert nicht: %s", self.directory
|
||||
# Verzeichnis auflösen
|
||||
directory = self._resolve_directory(config)
|
||||
|
||||
# Fehler-Liste zurücksetzen
|
||||
self.errors = []
|
||||
|
||||
# Extraktion durchführen
|
||||
artifacts, errors, skipped = self._do_extract(directory)
|
||||
|
||||
# Fehler speichern (für Legacy-Kompatibilität)
|
||||
self.errors = errors
|
||||
|
||||
if use_legacy:
|
||||
# Legacy-Rückgabe: nur die Artefaktliste
|
||||
return artifacts
|
||||
else:
|
||||
# Neues Interface: ExtractionResult (konvertiere lokale Fehler)
|
||||
base_errors = [e.to_base_error() for e in errors]
|
||||
return ExtractionResult(
|
||||
artifacts=artifacts,
|
||||
errors=base_errors,
|
||||
skipped=skipped,
|
||||
)
|
||||
self.errors.append(
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Markdown/Text-Quellen unterstützen inkrementelle Verarbeitung.
|
||||
|
||||
Über Content-Hashes werden nur geänderte Dateien erneut verarbeitet.
|
||||
|
||||
Returns:
|
||||
True – inkrementelle Verarbeitung wird unterstützt.
|
||||
"""
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inkrementelle Verarbeitung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def set_known_hashes(self, hashes: dict[str, str]) -> None:
|
||||
"""Setzt die bekannten Datei-Hashes für inkrementelle Verarbeitung.
|
||||
|
||||
Args:
|
||||
hashes: Mapping von Dateipfad (str) zu Content-Hash.
|
||||
Dateien mit unverändertem Hash werden übersprungen.
|
||||
"""
|
||||
self._known_hashes = dict(hashes)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Implementierung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_directory(self, config: SourceConfig | None) -> Path:
|
||||
"""Löst das Quellverzeichnis aus config oder self._directory auf.
|
||||
|
||||
Args:
|
||||
config: Optionale SourceConfig mit params["directory"].
|
||||
|
||||
Returns:
|
||||
Path zum Quellverzeichnis.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn weder config noch self._directory gesetzt ist.
|
||||
"""
|
||||
if config is not None:
|
||||
dir_param = config.params.get("directory", "")
|
||||
if dir_param:
|
||||
return Path(dir_param)
|
||||
|
||||
if self._directory is not None:
|
||||
return self._directory
|
||||
|
||||
raise ValueError(
|
||||
"Kein Verzeichnis angegeben: "
|
||||
"weder config.params['directory'] noch directory-Parameter gesetzt."
|
||||
)
|
||||
|
||||
def _do_extract(
|
||||
self, directory: Path
|
||||
) -> tuple[list[Artifact], list[SourceError], int]:
|
||||
"""Führt die eigentliche Extraktion durch.
|
||||
|
||||
Args:
|
||||
directory: Das zu scannende Verzeichnis.
|
||||
|
||||
Returns:
|
||||
Tuple von (artifacts, errors, skipped_count).
|
||||
"""
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
skipped = 0
|
||||
|
||||
# Verzeichnis-Validierung
|
||||
if not directory.exists():
|
||||
logger.warning(
|
||||
"Quellverzeichnis existiert nicht: %s", directory
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
file_path=str(self.directory),
|
||||
error=f"Verzeichnis existiert nicht: {self.directory}",
|
||||
file_path=str(directory),
|
||||
error=f"Verzeichnis existiert nicht: {directory}",
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
return artifacts
|
||||
return artifacts, errors, skipped
|
||||
|
||||
if not self.directory.is_dir():
|
||||
if not directory.is_dir():
|
||||
logger.warning(
|
||||
"Quellpfad ist kein Verzeichnis: %s", self.directory
|
||||
"Quellpfad ist kein Verzeichnis: %s", directory
|
||||
)
|
||||
self.errors.append(
|
||||
errors.append(
|
||||
SourceError(
|
||||
file_path=str(self.directory),
|
||||
error=f"Pfad ist kein Verzeichnis: {self.directory}",
|
||||
file_path=str(directory),
|
||||
error=f"Pfad ist kein Verzeichnis: {directory}",
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
return artifacts
|
||||
return artifacts, errors, skipped
|
||||
|
||||
md_files = sorted(self.directory.rglob("*.md"))
|
||||
# Alle unterstützten Dateien finden
|
||||
all_files = sorted(
|
||||
f
|
||||
for f in directory.rglob("*")
|
||||
if f.is_file() and f.suffix.lower() in _SUPPORTED_EXTENSIONS
|
||||
)
|
||||
|
||||
for md_file in md_files:
|
||||
try:
|
||||
artifact = parse_frontmatter(md_file)
|
||||
for file_path in all_files:
|
||||
# Inkrementelle Verarbeitung: Hash-Check
|
||||
if self._known_hashes:
|
||||
file_key = str(file_path)
|
||||
current_hash = _compute_file_hash(file_path)
|
||||
known_hash = self._known_hashes.get(file_key)
|
||||
if known_hash == current_hash:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Datei verarbeiten
|
||||
artifact = self._extract_file(file_path, errors)
|
||||
if artifact is not None:
|
||||
artifacts.append(artifact)
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
"Überspringe Datei ohne gültiges Frontmatter: %s – %s",
|
||||
md_file,
|
||||
e,
|
||||
)
|
||||
self.errors.append(
|
||||
SourceError(
|
||||
file_path=str(md_file),
|
||||
error=str(e),
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Lesen der Datei: %s – %s", md_file, e
|
||||
)
|
||||
self.errors.append(
|
||||
SourceError(
|
||||
file_path=str(md_file),
|
||||
error=str(e),
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
|
||||
return artifacts
|
||||
return artifacts, errors, skipped
|
||||
|
||||
def _extract_file(
|
||||
self, file_path: Path, errors: list[SourceError]
|
||||
) -> Artifact | None:
|
||||
"""Extrahiert ein einzelnes Artefakt aus einer Datei.
|
||||
|
||||
.md-Dateien: Parsed mit parse_frontmatter (erwartet Frontmatter).
|
||||
.txt-Dateien: Liest als Plain-Text und erstellt minimales Artefakt.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
errors: Fehlerliste zum Ergänzen bei Problemen.
|
||||
|
||||
Returns:
|
||||
Artifact oder None bei Fehler.
|
||||
"""
|
||||
try:
|
||||
if file_path.suffix.lower() == ".txt":
|
||||
return self._extract_txt_file(file_path)
|
||||
else:
|
||||
return parse_frontmatter(file_path)
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
"Überspringe Datei ohne gültiges Frontmatter: %s – %s",
|
||||
file_path,
|
||||
e,
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
file_path=str(file_path),
|
||||
error=str(e),
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
return None
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Lesen der Datei: %s – %s", file_path, e
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
file_path=str(file_path),
|
||||
error=str(e),
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
return None
|
||||
|
||||
def _extract_txt_file(self, file_path: Path) -> Artifact:
|
||||
"""Extrahiert ein Artefakt aus einer .txt-Datei.
|
||||
|
||||
Erstellt ein minimales Artefakt ohne Frontmatter – Titel wird
|
||||
aus dem Dateinamen abgeleitet.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur .txt-Datei.
|
||||
|
||||
Returns:
|
||||
Artifact mit minimalem Metadata.
|
||||
"""
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
title = file_path.stem.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type="note",
|
||||
title=title,
|
||||
source={"type": "file", "file": str(file_path)},
|
||||
)
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=file_path,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Legacy-Kompatibilität: Attribut-Zugriff
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def directory(self) -> Path | None:
|
||||
"""Zugriff auf das konfigurierte Verzeichnis (Legacy-Kompatibilität)."""
|
||||
return self._directory
|
||||
|
||||
@directory.setter
|
||||
def directory(self, value: Path) -> None:
|
||||
"""Setzt das Verzeichnis (Legacy-Kompatibilität)."""
|
||||
self._directory = value
|
||||
|
||||
@@ -0,0 +1,407 @@
|
||||
"""PDF- und OCR-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Implementiert die Extraktion von Text aus PDF-Dateien (eingebetteter Text
|
||||
und OCR-Fallback für gescannte Seiten) sowie die direkte OCR-Verarbeitung
|
||||
von Bilddateien (PNG, JPG, JPEG).
|
||||
|
||||
OCR-Strategie (Priorität):
|
||||
1. Kiro Vision – Bilder werden an den AI-Agenten gesendet (primär)
|
||||
2. Tesseract (optional, spätere Alternative für CI/Batch)
|
||||
|
||||
Requirements: 4.2, 4.3, 4.4, 4.5, 4.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata, compute_content_hash
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Unterstützte Dateierweiterungen
|
||||
PDF_EXTENSIONS = {".pdf"}
|
||||
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg"}
|
||||
SUPPORTED_EXTENSIONS = PDF_EXTENSIONS | IMAGE_EXTENSIONS
|
||||
|
||||
# OCR-Sprachkonfiguration
|
||||
OCR_LANGUAGES = "deu+eng"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDF-Textextraktion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_pdf_text(file_path: Path) -> tuple[str, bool]:
|
||||
"""Extrahiert Text aus einer PDF-Datei.
|
||||
|
||||
Versucht eingebetteten Text via pypdf zu extrahieren. Falls Seiten
|
||||
keinen Text enthalten (gescannte PDFs), wird ein OCR-Fallback benötigt.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur PDF-Datei.
|
||||
|
||||
Returns:
|
||||
Tuple (extrahierter_text, needs_ocr).
|
||||
needs_ocr ist True wenn mindestens eine Seite keinen Text enthält.
|
||||
|
||||
Raises:
|
||||
ImportError: Wenn keine PDF-Bibliothek verfügbar ist.
|
||||
OSError: Bei Dateizugriffsfehlern.
|
||||
"""
|
||||
try:
|
||||
from pypdf import PdfReader # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
try:
|
||||
from PyPDF2 import PdfReader # type: ignore[import-untyped, no-redef]
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"Keine PDF-Bibliothek verfügbar. "
|
||||
"Installiere 'pypdf' oder 'PyPDF2': pip install pypdf"
|
||||
)
|
||||
|
||||
reader = PdfReader(str(file_path))
|
||||
pages_text: list[str] = []
|
||||
pages_without_text: list[int] = []
|
||||
|
||||
for i, page in enumerate(reader.pages):
|
||||
text = page.extract_text() or ""
|
||||
text = text.strip()
|
||||
if text:
|
||||
pages_text.append(text)
|
||||
else:
|
||||
pages_without_text.append(i)
|
||||
|
||||
full_text = "\n\n".join(pages_text)
|
||||
needs_ocr = len(pages_without_text) > 0
|
||||
|
||||
if needs_ocr:
|
||||
logger.info(
|
||||
"PDF %s: %d von %d Seiten ohne Text (OCR benötigt für Seiten: %s)",
|
||||
file_path.name,
|
||||
len(pages_without_text),
|
||||
len(reader.pages),
|
||||
pages_without_text,
|
||||
)
|
||||
|
||||
return full_text, needs_ocr
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OCR-Engine (Kiro Vision als primärer Provider)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _perform_ocr(file_path: Path, languages: str = OCR_LANGUAGES) -> tuple[str, bool]:
|
||||
"""Führt OCR auf einer Bild- oder PDF-Datei durch.
|
||||
|
||||
Primärer Provider: Kiro Vision (über kiro_client-Integration).
|
||||
Fallback: Stub, der ocr_failed=True zurückgibt wenn Kiro nicht verfügbar.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Bild- oder PDF-Datei.
|
||||
languages: Sprachkonfiguration (default: deu+eng).
|
||||
|
||||
Returns:
|
||||
Tuple (extrahierter_text, ocr_failed).
|
||||
ocr_failed ist True wenn OCR fehlgeschlagen ist.
|
||||
"""
|
||||
# Versuche Kiro Vision Client
|
||||
try:
|
||||
from monorepo.knowledge.integrations.kiro_client import ( # type: ignore[import-untyped]
|
||||
KiroClient,
|
||||
)
|
||||
|
||||
client = KiroClient()
|
||||
text = client.ocr(file_path, languages=languages)
|
||||
if text and text.strip():
|
||||
return text.strip(), False
|
||||
else:
|
||||
logger.warning(
|
||||
"Kiro Vision lieferte keinen Text für: %s", file_path
|
||||
)
|
||||
return "", True
|
||||
except ImportError:
|
||||
logger.info(
|
||||
"kiro_client nicht verfügbar, versuche Tesseract-Fallback für: %s",
|
||||
file_path,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Kiro Vision OCR fehlgeschlagen für %s: %s", file_path, e
|
||||
)
|
||||
|
||||
# Fallback: Tesseract (optional)
|
||||
try:
|
||||
import pytesseract # type: ignore[import-untyped]
|
||||
from PIL import Image # type: ignore[import-untyped]
|
||||
|
||||
if file_path.suffix.lower() in IMAGE_EXTENSIONS:
|
||||
img = Image.open(file_path)
|
||||
text = pytesseract.image_to_string(img, lang=languages.replace("+", "+"))
|
||||
if text and text.strip():
|
||||
return text.strip(), False
|
||||
else:
|
||||
# Für PDFs: Tesseract kann nur Bilder verarbeiten
|
||||
logger.info(
|
||||
"Tesseract-OCR nur für Bilddateien verfügbar, nicht für PDF-Seiten: %s",
|
||||
file_path,
|
||||
)
|
||||
return "", True
|
||||
except ImportError:
|
||||
logger.info(
|
||||
"Weder kiro_client noch pytesseract verfügbar für OCR: %s",
|
||||
file_path,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Tesseract OCR fehlgeschlagen für %s: %s", file_path, e)
|
||||
|
||||
return "", True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDFSource Strategie
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PDFSource(SourceStrategy):
|
||||
"""Quellstrategie für PDF- und Bilddateien.
|
||||
|
||||
Verarbeitet:
|
||||
- PDF-Dateien: Eingebetteten Text extrahieren, OCR-Fallback für gescannte Seiten
|
||||
- Bilddateien (PNG, JPG, JPEG): Direkt an OCR weiterleiten
|
||||
|
||||
Setzt `ocr_failed: true` im Artefakt-Frontmatter falls OCR fehlschlägt.
|
||||
Behält Referenz auf Originaldatei in `source.file`.
|
||||
"""
|
||||
|
||||
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
|
||||
"""Extrahiert Artefakte aus PDF- und Bilddateien.
|
||||
|
||||
Args:
|
||||
config: SourceConfig mit params["path"] oder params["directory"].
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
result = ExtractionResult()
|
||||
|
||||
# Pfad aus Konfiguration ermitteln
|
||||
source_path = self._resolve_source_path(config)
|
||||
if source_path is None:
|
||||
result.errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message="Kein gültiger Pfad in config.params['path'] oder config.params['directory']",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
# Dateien sammeln
|
||||
files = self._collect_files(source_path)
|
||||
if not files:
|
||||
logger.info("Keine unterstützten Dateien gefunden in: %s", source_path)
|
||||
return result
|
||||
|
||||
# Dateien verarbeiten
|
||||
for file_path in files:
|
||||
try:
|
||||
artifact = self._process_file(file_path, config, context)
|
||||
if artifact is not None:
|
||||
result.artifacts.append(artifact)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Fehler bei der Verarbeitung von %s: %s", file_path, e
|
||||
)
|
||||
result.errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler bei {file_path.name}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=True,
|
||||
artifact_path=str(file_path),
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""PDF-Source unterstützt inkrementelle Verarbeitung via Content-Hash."""
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Hilfsmethoden
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_source_path(self, config: SourceConfig) -> Path | None:
|
||||
"""Ermittelt den Quellpfad aus der Konfiguration."""
|
||||
path_str = config.params.get("path") or config.params.get("directory") or ""
|
||||
if not path_str:
|
||||
return None
|
||||
source_path = Path(path_str)
|
||||
if not source_path.exists():
|
||||
return None
|
||||
return source_path
|
||||
|
||||
def _collect_files(self, source_path: Path) -> list[Path]:
|
||||
"""Sammelt alle unterstützten Dateien aus dem Quellpfad."""
|
||||
if source_path.is_file():
|
||||
if source_path.suffix.lower() in SUPPORTED_EXTENSIONS:
|
||||
return [source_path]
|
||||
return []
|
||||
|
||||
# Verzeichnis rekursiv durchsuchen
|
||||
files: list[Path] = []
|
||||
for ext in SUPPORTED_EXTENSIONS:
|
||||
files.extend(source_path.rglob(f"*{ext}"))
|
||||
return sorted(files)
|
||||
|
||||
def _process_file(
|
||||
self, file_path: Path, config: SourceConfig, context: str
|
||||
) -> Artifact | None:
|
||||
"""Verarbeitet eine einzelne PDF- oder Bilddatei.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
config: Die Quellkonfiguration.
|
||||
context: Der Zielkontext.
|
||||
|
||||
Returns:
|
||||
Ein Artifact oder None bei Fehler.
|
||||
"""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix in PDF_EXTENSIONS:
|
||||
return self._process_pdf(file_path, config, context)
|
||||
elif suffix in IMAGE_EXTENSIONS:
|
||||
return self._process_image(file_path, config, context)
|
||||
else:
|
||||
return None
|
||||
|
||||
def _process_pdf(
|
||||
self, file_path: Path, config: SourceConfig, context: str
|
||||
) -> Artifact:
|
||||
"""Verarbeitet eine PDF-Datei.
|
||||
|
||||
1. Versucht eingebetteten Text zu extrahieren
|
||||
2. Falls Seiten ohne Text: OCR-Fallback
|
||||
3. Setzt ocr_failed bei Fehlschlag
|
||||
"""
|
||||
ocr_failed = False
|
||||
text = ""
|
||||
|
||||
try:
|
||||
embedded_text, needs_ocr = _extract_pdf_text(file_path)
|
||||
text = embedded_text
|
||||
|
||||
if needs_ocr and not embedded_text:
|
||||
# Komplett gescanntes PDF – OCR für das gesamte Dokument
|
||||
ocr_text, ocr_failed = _perform_ocr(file_path)
|
||||
if ocr_text:
|
||||
text = ocr_text
|
||||
elif needs_ocr and embedded_text:
|
||||
# Teilweise gescannt – Text vorhanden, OCR nur für fehlende Seiten
|
||||
# Für jetzt verwenden wir den vorhandenen Text
|
||||
logger.info(
|
||||
"PDF %s: Eingebetteter Text vorhanden, gescannte Seiten übersprungen",
|
||||
file_path.name,
|
||||
)
|
||||
except ImportError as e:
|
||||
logger.warning("PDF-Bibliothek nicht verfügbar: %s", e)
|
||||
# Versuche OCR als Fallback
|
||||
ocr_text, ocr_failed = _perform_ocr(file_path)
|
||||
text = ocr_text
|
||||
|
||||
return self._create_artifact(
|
||||
file_path=file_path,
|
||||
text=text,
|
||||
context=context,
|
||||
source_type="pdf",
|
||||
ocr_failed=ocr_failed,
|
||||
)
|
||||
|
||||
def _process_image(
|
||||
self, file_path: Path, config: SourceConfig, context: str
|
||||
) -> Artifact:
|
||||
"""Verarbeitet eine Bilddatei via OCR.
|
||||
|
||||
Bilddateien werden direkt an die OCR-Engine weitergeleitet.
|
||||
"""
|
||||
text, ocr_failed = _perform_ocr(file_path)
|
||||
|
||||
return self._create_artifact(
|
||||
file_path=file_path,
|
||||
text=text,
|
||||
context=context,
|
||||
source_type="image",
|
||||
ocr_failed=ocr_failed,
|
||||
)
|
||||
|
||||
def _create_artifact(
|
||||
self,
|
||||
file_path: Path,
|
||||
text: str,
|
||||
context: str,
|
||||
source_type: str,
|
||||
ocr_failed: bool,
|
||||
) -> Artifact:
|
||||
"""Erstellt ein Artifact aus den extrahierten Daten.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Originaldatei.
|
||||
text: Der extrahierte Text.
|
||||
context: Der Zielkontext.
|
||||
source_type: "pdf" oder "image".
|
||||
ocr_failed: Ob OCR fehlgeschlagen ist.
|
||||
|
||||
Returns:
|
||||
Ein vollständiges Artifact.
|
||||
"""
|
||||
# Titel aus Dateinamen ableiten
|
||||
title = file_path.stem.replace("-", " ").replace("_", " ").strip()
|
||||
if not title:
|
||||
title = file_path.name
|
||||
|
||||
# Content-Hash berechnen
|
||||
content_hash = compute_content_hash(text) if text else ""
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type="reference",
|
||||
title=title,
|
||||
tags=[source_type],
|
||||
source_context=context,
|
||||
created=date.today(),
|
||||
updated=date.today(),
|
||||
shareable=False,
|
||||
content_hash=content_hash,
|
||||
source={
|
||||
"type": source_type,
|
||||
"file": str(file_path),
|
||||
},
|
||||
category="references",
|
||||
ocr_failed=ocr_failed,
|
||||
)
|
||||
|
||||
# Markdown-Inhalt zusammenbauen
|
||||
content = f"# {title}\n\n{text}\n" if text else f"# {title}\n\n"
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=file_path,
|
||||
)
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Wissensdatenbank-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Adapter für `bahn/wissensdatenbank/output/` – liest Markdown-Artefakte
|
||||
als Read-Only-Referenzen in den bahn/knowledge/ YAML-Index ein, ohne
|
||||
die Originaldateien zu verändern oder dem Google OKF-Format aufzuzwingen.
|
||||
|
||||
Requirements: 13.1, 13.2, 13.3, 13.4, 13.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["WissensdatenbankSource"]
|
||||
|
||||
|
||||
# Unterstützte Dateiendungen in der Wissensdatenbank
|
||||
_SUPPORTED_EXTENSIONS = {".md", ".txt"}
|
||||
|
||||
|
||||
def _compute_file_hash(file_path: Path) -> str:
|
||||
"""Berechnet den SHA-256-Hash des Dateiinhalts.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
|
||||
Returns:
|
||||
Hash-String im Format "sha256:<hex>".
|
||||
"""
|
||||
content = file_path.read_bytes()
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
class WissensdatenbankSource(SourceStrategy):
|
||||
"""Quellstrategie: Read-Only-Referenzen aus bahn/wissensdatenbank/output/.
|
||||
|
||||
Scannt das Wissensdatenbank-Output-Verzeichnis nach Markdown-Dateien
|
||||
und erstellt Artifact-Objekte mit `source: wissensdatenbank`-Metadaten.
|
||||
|
||||
**Kernverhalten:**
|
||||
- Originaldateien werden NICHT modifiziert (Read-Only-Referenz)
|
||||
- Kein Google-OKF-Reformat (kein Splitting, keine index.md-Generierung)
|
||||
- Artefakte werden als Referenzen im Index geführt (source: wissensdatenbank)
|
||||
- Inkrementelle Verarbeitung via Content-Hash
|
||||
|
||||
Requirements: 13.1, 13.2, 13.3, 13.4, 13.5
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialisiert die WissensdatenbankSource."""
|
||||
self._known_hashes: dict[str, str] = {}
|
||||
|
||||
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
|
||||
"""Extrahiert Read-Only-Referenzen aus dem Wissensdatenbank-Output.
|
||||
|
||||
Scannt `params["output_path"]` rekursiv nach .md/.txt-Dateien und
|
||||
erstellt Artifact-Objekte ohne die Originaldateien zu verändern.
|
||||
|
||||
Args:
|
||||
config: Die Quellkonfiguration mit `params["output_path"]`.
|
||||
context: Der Zielkontext (typischerweise "bahn").
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
output_path_str = config.params.get("output_path", "")
|
||||
if not output_path_str:
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message="Kein 'output_path' in params konfiguriert.",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
output_path = Path(output_path_str)
|
||||
|
||||
if not output_path.exists():
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="connection",
|
||||
message=f"Wissensdatenbank-Verzeichnis existiert nicht: {output_path}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
if not output_path.is_dir():
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Pfad ist kein Verzeichnis: {output_path}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
skipped = 0
|
||||
|
||||
# Alle unterstützten Dateien finden (rekursiv)
|
||||
all_files = sorted(
|
||||
f
|
||||
for f in output_path.rglob("*")
|
||||
if f.is_file() and f.suffix.lower() in _SUPPORTED_EXTENSIONS
|
||||
)
|
||||
|
||||
for file_path in all_files:
|
||||
# Inkrementelle Verarbeitung: Hash-Check
|
||||
file_key = str(file_path)
|
||||
try:
|
||||
current_hash = _compute_file_hash(file_path)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Lesen der Datei: %s – %s", file_path, e
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler beim Lesen: {file_path} – {e}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
artifact_path=file_key,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
known_hash = self._known_hashes.get(file_key)
|
||||
if known_hash == current_hash:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Datei lesen und als Read-Only-Referenz erstellen
|
||||
try:
|
||||
artifact = self._create_reference_artifact(
|
||||
file_path, current_hash, context
|
||||
)
|
||||
artifacts.append(artifact)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Verarbeiten: %s – %s", file_path, e
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler beim Verarbeiten: {file_path} – {e}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
artifact_path=file_key,
|
||||
)
|
||||
)
|
||||
|
||||
return ExtractionResult(
|
||||
artifacts=artifacts,
|
||||
errors=errors,
|
||||
skipped=skipped,
|
||||
)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Wissensdatenbank unterstützt inkrementelle Verarbeitung via Hash.
|
||||
|
||||
Returns:
|
||||
True – inkrementelle Verarbeitung wird unterstützt.
|
||||
"""
|
||||
return True
|
||||
|
||||
def set_known_hashes(self, hashes: dict[str, str]) -> None:
|
||||
"""Setzt die bekannten Datei-Hashes für inkrementelle Verarbeitung.
|
||||
|
||||
Args:
|
||||
hashes: Mapping von Dateipfad (str) zu Content-Hash.
|
||||
Dateien mit unverändertem Hash werden übersprungen.
|
||||
"""
|
||||
self._known_hashes = dict(hashes)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Implementierung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _create_reference_artifact(
|
||||
self, file_path: Path, content_hash: str, context: str
|
||||
) -> Artifact:
|
||||
"""Erstellt ein Read-Only-Referenz-Artefakt aus einer Wissensdatenbank-Datei.
|
||||
|
||||
Die Originaldatei wird gelesen aber NICHT verändert. Das Artefakt
|
||||
erhält `source: {"type": "wissensdatenbank", "path": <original_path>}`
|
||||
als Metadaten.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Originaldatei in wissensdatenbank/output/.
|
||||
content_hash: Bereits berechneter SHA-256-Hash.
|
||||
context: Der Zielkontext (typischerweise "bahn").
|
||||
|
||||
Returns:
|
||||
Artifact mit Read-Only-Referenz-Metadaten.
|
||||
"""
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
|
||||
# Titel aus Dateiname ableiten (ersetzt Bindestriche/Underscores)
|
||||
title = file_path.stem.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type="reference",
|
||||
title=title,
|
||||
source_context=context,
|
||||
source={"type": "wissensdatenbank", "path": str(file_path)},
|
||||
category="references",
|
||||
content_hash=content_hash,
|
||||
created=date.today(),
|
||||
)
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=file_path,
|
||||
)
|
||||
@@ -4,6 +4,8 @@ Basiert auf der DB-Wissensdatenbank-ETL-Architektur und bietet:
|
||||
- Ingestion von Wissensartefakten aus verschiedenen Quellen
|
||||
- Progressive-Disclosure-Index (kompakte YAML-Metadaten, Schicht 1)
|
||||
- Volltextsuche mit Scope-basierter Zugriffskontrolle und Relevanz-Sortierung
|
||||
- Kontextübergreifende Suche über alle Knowledge-Folder _index.yaml-Dateien
|
||||
- Agent-Context-Injection: max. 10 relevante Artefakt-Pfade pro Anfrage
|
||||
- Graph-Verknüpfungen zwischen Artefakten
|
||||
- Timeout-Handling für Suchanfragen (5 Sekunden)
|
||||
"""
|
||||
@@ -26,6 +28,12 @@ logger = logging.getLogger(__name__)
|
||||
# Default-Timeout für Suchanfragen in Sekunden
|
||||
SEARCH_TIMEOUT_SECONDS: float = 5.0
|
||||
|
||||
# Max. Artefakt-Pfade für Agent-Context-Injection
|
||||
MAX_CONTEXT_INJECTION_RESULTS: int = 10
|
||||
|
||||
# Bekannte Kontext-Knowledge-Folder (relativ zum Monorepo-Root)
|
||||
CONTEXT_KNOWLEDGE_FOLDERS: list[str] = ["bahn", "dhive", "privat"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
@@ -48,6 +56,24 @@ class SearchResult:
|
||||
partial_results: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContextInjectionResult:
|
||||
"""Ergebnis der Agent-Context-Injection.
|
||||
|
||||
Enthält die relevantesten Artefakt-Pfade für den Agent-Prompt,
|
||||
sortiert nach Relevanz-Score. Maximal MAX_CONTEXT_INJECTION_RESULTS Einträge.
|
||||
|
||||
Attributes:
|
||||
paths: Liste der relevantesten Artefakt-Pfade (max. 10).
|
||||
entries: Die zugehörigen SearchResult-Objekte (Progressive Disclosure Schicht 1).
|
||||
partial_results: True wenn die Suche wegen Timeout abgebrochen wurde.
|
||||
"""
|
||||
|
||||
paths: list[str] = field(default_factory=list)
|
||||
entries: list[SearchResult] = field(default_factory=list)
|
||||
partial_results: bool = False
|
||||
|
||||
|
||||
def _compute_relevance(query_lower: str, entry: IndexEntry) -> float:
|
||||
"""Berechnet den Relevanz-Score für einen Index-Eintrag.
|
||||
|
||||
@@ -115,6 +141,11 @@ class KnowledgeStore:
|
||||
- Schicht 2 (Dokument): Vollständige Markdown-Datei mit YAML-Frontmatter
|
||||
|
||||
Agenten lesen zuerst den Index und laden nur bei Bedarf einzelne Dokumente.
|
||||
|
||||
Kontextübergreifende Suche:
|
||||
- Durchsucht alle Knowledge-Folder _index.yaml-Dateien (bahn, dhive, privat)
|
||||
- Respektiert ctx-guard-Sicherheitsgrenzen (nur autorisierte Scopes)
|
||||
- Agent-Context-Injection: max. 10 relevante Pfade pro Anfrage
|
||||
"""
|
||||
|
||||
INDEX_FILENAME = "_index.yaml"
|
||||
@@ -124,7 +155,7 @@ class KnowledgeStore:
|
||||
|
||||
Args:
|
||||
base_path: Basispfad des Wissensspeichers
|
||||
(z.B. shared/knowledge-store/).
|
||||
(z.B. shared/knowledge-store/ oder Monorepo-Root).
|
||||
scope_config: Konfiguration des Scope-Mappings
|
||||
(Arbeitskontext → Scope).
|
||||
"""
|
||||
@@ -350,3 +381,222 @@ class KnowledgeStore:
|
||||
|
||||
# Datei zurückschreiben (mit aktualisiertem Frontmatter)
|
||||
write_artifact(artifact, self.base_path)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Kontextübergreifende Suche über Knowledge-Folder
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def _load_knowledge_folder_indices(self) -> list[YAMLIndex]:
|
||||
"""Lädt alle _index.yaml-Dateien aus den Knowledge-Foldern.
|
||||
|
||||
Durchsucht die bekannten Kontext-Knowledge-Folder (bahn, dhive, privat)
|
||||
und lädt deren _index.yaml als YAMLIndex-Instanzen.
|
||||
|
||||
Returns:
|
||||
Liste geladener YAMLIndex-Instanzen (nur existierende Dateien).
|
||||
"""
|
||||
indices: list[YAMLIndex] = []
|
||||
|
||||
for context_name in CONTEXT_KNOWLEDGE_FOLDERS:
|
||||
index_path = self.base_path / context_name / "knowledge" / self.INDEX_FILENAME
|
||||
if index_path.exists():
|
||||
try:
|
||||
idx = YAMLIndex(index_path)
|
||||
idx.load()
|
||||
indices.append(idx)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Konnte Knowledge-Folder-Index nicht laden: %s (%s)",
|
||||
index_path,
|
||||
e,
|
||||
)
|
||||
|
||||
return indices
|
||||
|
||||
def search_cross_context(
|
||||
self,
|
||||
query: str,
|
||||
allowed_scopes: list[str],
|
||||
timeout: float | None = None,
|
||||
) -> list[SearchResult]:
|
||||
"""Kontextübergreifende Suche über alle Knowledge-Folder _index.yaml-Dateien.
|
||||
|
||||
Durchsucht sowohl den eigenen Index als auch alle Kontext-Knowledge-Folder
|
||||
(_index.yaml in bahn/knowledge/, dhive/knowledge/, privat/knowledge/).
|
||||
Respektiert ctx-guard-Sicherheitsgrenzen: nur Artefakte aus autorisierten
|
||||
Scopes werden zurückgegeben.
|
||||
|
||||
Progressive Disclosure (Schicht 1): Die Suche operiert ausschließlich
|
||||
auf dem YAML-Index (kompakte Metadaten). Vollständige Dokumente werden
|
||||
nur bei explizitem Nachladen über den Pfad bereitgestellt.
|
||||
|
||||
Args:
|
||||
query: Suchbegriff für die Volltextsuche.
|
||||
allowed_scopes: Liste der Scopes, aus denen Ergebnisse
|
||||
geliefert werden dürfen (ctx-guard-Autorisierung).
|
||||
timeout: Timeout in Sekunden (None = SEARCH_TIMEOUT_SECONDS).
|
||||
|
||||
Returns:
|
||||
Liste von SearchResult-Objekten, sortiert nach Relevanz
|
||||
(höchster Score zuerst). Enthält Ergebnisse aus dem eigenen
|
||||
Index UND allen Knowledge-Folder-Indices.
|
||||
|
||||
Note:
|
||||
Requirements: 11.1, 11.3, 11.5
|
||||
"""
|
||||
if not query:
|
||||
return []
|
||||
|
||||
effective_timeout = timeout if timeout is not None else SEARCH_TIMEOUT_SECONDS
|
||||
query_lower = query.lower()
|
||||
results: list[SearchResult] = []
|
||||
seen_ids: set[str] = set()
|
||||
timed_out = False
|
||||
start_time = time.monotonic()
|
||||
|
||||
# 1. Eigenen Index durchsuchen (shared KnowledgeStore)
|
||||
for entry in self.index.entries.values():
|
||||
elapsed = time.monotonic() - start_time
|
||||
if elapsed >= effective_timeout:
|
||||
timed_out = True
|
||||
break
|
||||
|
||||
if entry.scope not in allowed_scopes:
|
||||
continue
|
||||
|
||||
searchable_text = " ".join([
|
||||
entry.title, entry.id, entry.summary, " ".join(entry.tags),
|
||||
]).lower()
|
||||
|
||||
if query_lower in searchable_text:
|
||||
relevance = _compute_relevance(query_lower, entry)
|
||||
results.append(SearchResult(
|
||||
entry=entry, relevance_score=relevance, partial_results=False,
|
||||
))
|
||||
seen_ids.add(entry.id)
|
||||
|
||||
# 2. Knowledge-Folder-Indices durchsuchen
|
||||
if not timed_out:
|
||||
folder_indices = self._load_knowledge_folder_indices()
|
||||
|
||||
for folder_index in folder_indices:
|
||||
if timed_out:
|
||||
break
|
||||
|
||||
for entry in folder_index.entries.values():
|
||||
elapsed = time.monotonic() - start_time
|
||||
if elapsed >= effective_timeout:
|
||||
timed_out = True
|
||||
break
|
||||
|
||||
# Scope-Filter: Nur autorisierte Scopes
|
||||
if entry.scope not in allowed_scopes:
|
||||
continue
|
||||
|
||||
# Deduplizierung: Einträge die bereits im eigenen Index sind
|
||||
if entry.id in seen_ids:
|
||||
continue
|
||||
|
||||
searchable_text = " ".join([
|
||||
entry.title, entry.id, entry.summary, " ".join(entry.tags),
|
||||
]).lower()
|
||||
|
||||
if query_lower in searchable_text:
|
||||
relevance = _compute_relevance(query_lower, entry)
|
||||
results.append(SearchResult(
|
||||
entry=entry, relevance_score=relevance, partial_results=False,
|
||||
))
|
||||
seen_ids.add(entry.id)
|
||||
|
||||
# Bei Timeout: Alle Ergebnisse als partial markieren
|
||||
if timed_out:
|
||||
for result in results:
|
||||
result.partial_results = True
|
||||
|
||||
# Nach Relevanz sortieren (höchster Score zuerst)
|
||||
results.sort(key=lambda r: r.relevance_score, reverse=True)
|
||||
|
||||
return results
|
||||
|
||||
def get_context_injection(
|
||||
self,
|
||||
query: str,
|
||||
allowed_scopes: list[str],
|
||||
max_results: int | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> ContextInjectionResult:
|
||||
"""Agent-Context-Injection: Liefert die relevantesten Artefakt-Pfade.
|
||||
|
||||
Identifiziert die relevantesten Artefakte aus den YAML-Indices aller
|
||||
autorisierten Kontexte und liefert deren Pfade als Kontextinformation
|
||||
für den Agent-Prompt.
|
||||
|
||||
Progressive Disclosure:
|
||||
- Schicht 1: YAML-Index-Daten werden für die Relevanzprüfung verwendet.
|
||||
- Schicht 2: Nur die Pfade werden zurückgegeben – das vollständige
|
||||
Dokument wird vom Agenten bei Bedarf nachgeladen.
|
||||
|
||||
Args:
|
||||
query: Suchbegriff oder Task-Beschreibung für Relevanzprüfung.
|
||||
allowed_scopes: Autorisierte Scopes (ctx-guard-Filterung).
|
||||
max_results: Maximale Anzahl zurückgegebener Pfade
|
||||
(None = MAX_CONTEXT_INJECTION_RESULTS = 10).
|
||||
timeout: Timeout in Sekunden (None = SEARCH_TIMEOUT_SECONDS).
|
||||
|
||||
Returns:
|
||||
ContextInjectionResult mit max. 10 Pfaden, sortiert nach
|
||||
Relevanz-Score (höchster zuerst).
|
||||
|
||||
Note:
|
||||
Requirements: 11.2, 11.5, 11.6
|
||||
"""
|
||||
effective_max = max_results if max_results is not None else MAX_CONTEXT_INJECTION_RESULTS
|
||||
|
||||
# Kontextübergreifende Suche durchführen
|
||||
all_results = self.search_cross_context(query, allowed_scopes, timeout)
|
||||
|
||||
# Auf max. Ergebnisse beschränken
|
||||
top_results = all_results[:effective_max]
|
||||
|
||||
# Pfade extrahieren (nur Einträge mit gültigem Pfad)
|
||||
paths: list[str] = []
|
||||
filtered_results: list[SearchResult] = []
|
||||
for result in top_results:
|
||||
if result.entry.path:
|
||||
paths.append(result.entry.path)
|
||||
filtered_results.append(result)
|
||||
|
||||
partial = any(r.partial_results for r in top_results) if top_results else False
|
||||
|
||||
return ContextInjectionResult(
|
||||
paths=paths,
|
||||
entries=filtered_results,
|
||||
partial_results=partial,
|
||||
)
|
||||
|
||||
def load_artifact_content(self, artifact_path: str) -> str | None:
|
||||
"""Progressive Disclosure Schicht 2: Lädt den vollständigen Artefakt-Inhalt.
|
||||
|
||||
Wird vom Agenten aufgerufen, wenn nach der Relevanzprüfung (Schicht 1)
|
||||
der vollständige Dokumentinhalt benötigt wird.
|
||||
|
||||
Args:
|
||||
artifact_path: Relativer Pfad zum Artefakt (aus dem IndexEntry.path).
|
||||
|
||||
Returns:
|
||||
Der vollständige Dateiinhalt als String, oder None wenn die Datei
|
||||
nicht existiert oder nicht gelesen werden kann.
|
||||
|
||||
Note:
|
||||
Requirements: 11.3
|
||||
"""
|
||||
full_path = self.base_path / artifact_path
|
||||
if not full_path.exists():
|
||||
logger.debug("Artefakt-Datei nicht gefunden für Schicht-2-Zugriff: %s", full_path)
|
||||
return None
|
||||
|
||||
try:
|
||||
return full_path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError) as e:
|
||||
logger.warning("Konnte Artefakt nicht lesen: %s (%s)", full_path, e)
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
"""Property-basierte Tests für Artifact Serialization Round-Trip.
|
||||
|
||||
**Validates: Requirements 16.6, 1.5**
|
||||
|
||||
Property 1: Artifact Serialization Round-Trip
|
||||
- For any valid ArtifactMetadata object with content, writing it to a Markdown
|
||||
file with YAML frontmatter and then reading it back SHALL produce an identical
|
||||
ArtifactMetadata object (fields, types, and values match).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
ArtifactLink,
|
||||
ArtifactMetadata,
|
||||
_generate_frontmatter,
|
||||
parse_frontmatter,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_TYPES = st.sampled_from(
|
||||
["meeting", "decision", "project", "reference", "link", "inbox", "note"]
|
||||
)
|
||||
|
||||
|
||||
def _safe_text(min_size: int = 1, max_size: int = 60) -> st.SearchStrategy[str]:
|
||||
"""Non-empty text that avoids YAML-problematic leading characters.
|
||||
|
||||
Avoids characters that would break YAML parsing when used in frontmatter
|
||||
values (e.g., leading `#`, `{`, `[`, `|`, `>`, `*`, `&`, `!`, `%`, `@`).
|
||||
Also avoids control characters and bare newlines within a single field value.
|
||||
"""
|
||||
# Use a broad unicode alphabet but exclude problematic chars
|
||||
alphabet = st.characters(
|
||||
whitelist_categories=("L", "N", "P", "S", "Z"),
|
||||
blacklist_characters="\x00\r\n\t{}[]|>*&!%@#`\"'\\:",
|
||||
)
|
||||
return st.text(alphabet, min_size=min_size, max_size=max_size).filter(
|
||||
lambda t: t.strip() != ""
|
||||
)
|
||||
|
||||
|
||||
@st.composite
|
||||
def slugified_tag(draw: st.DrawFn) -> str:
|
||||
"""Generates a slugified tag: lowercase alphanumeric with hyphens."""
|
||||
return draw(
|
||||
st.from_regex(r"[a-z][a-z0-9\-]{0,12}[a-z0-9]", fullmatch=True)
|
||||
)
|
||||
|
||||
|
||||
@st.composite
|
||||
def source_context_strategy(draw: st.DrawFn) -> str:
|
||||
"""Valid source_context values."""
|
||||
return draw(st.sampled_from(["bahn", "dhive", "privat", ""]))
|
||||
|
||||
|
||||
@st.composite
|
||||
def optional_date(draw: st.DrawFn) -> date | None:
|
||||
"""Optional date in a reasonable range."""
|
||||
if draw(st.booleans()):
|
||||
return None
|
||||
return draw(st.dates(min_value=date(2000, 1, 1), max_value=date(2099, 12, 31)))
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_link(draw: st.DrawFn) -> ArtifactLink:
|
||||
"""Generates a valid ArtifactLink."""
|
||||
target = draw(
|
||||
st.from_regex(r"[a-z][a-z0-9\-/]{1,30}[a-z0-9]", fullmatch=True)
|
||||
)
|
||||
relation = draw(
|
||||
st.from_regex(r"[a-z][a-z_]{1,15}[a-z]", fullmatch=True)
|
||||
)
|
||||
return ArtifactLink(target=target, relation=relation)
|
||||
|
||||
|
||||
@st.composite
|
||||
def content_hash_strategy(draw: st.DrawFn) -> str:
|
||||
"""Generates a sha256-prefixed hex string or empty."""
|
||||
if draw(st.booleans()):
|
||||
return ""
|
||||
hex_chars = draw(
|
||||
st.text(
|
||||
alphabet=st.sampled_from("0123456789abcdef"),
|
||||
min_size=64,
|
||||
max_size=64,
|
||||
)
|
||||
)
|
||||
return f"sha256:{hex_chars}"
|
||||
|
||||
|
||||
@st.composite
|
||||
def source_dict_strategy(draw: st.DrawFn) -> dict[str, str]:
|
||||
"""Generates a source dict with string keys and values."""
|
||||
if draw(st.booleans()):
|
||||
return {}
|
||||
# Use a fixed set of common keys to avoid YAML issues
|
||||
keys = draw(
|
||||
st.lists(
|
||||
st.sampled_from(["type", "url", "space", "file", "id", "path"]),
|
||||
min_size=1,
|
||||
max_size=4,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
result: dict[str, str] = {}
|
||||
for key in keys:
|
||||
val = draw(
|
||||
st.from_regex(r"[a-zA-Z0-9\-_./]{1,40}", fullmatch=True)
|
||||
)
|
||||
result[key] = val
|
||||
return result
|
||||
|
||||
|
||||
@st.composite
|
||||
def people_strategy(draw: st.DrawFn) -> list[str]:
|
||||
"""Generates a list of people references."""
|
||||
return draw(
|
||||
st.lists(
|
||||
st.from_regex(r"\[\[people/[a-z\-]{2,20}\]\]", fullmatch=True),
|
||||
min_size=0,
|
||||
max_size=3,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@st.composite
|
||||
def projects_strategy(draw: st.DrawFn) -> list[str]:
|
||||
"""Generates a list of project references."""
|
||||
return draw(
|
||||
st.lists(
|
||||
st.from_regex(r"\[\[projects/[a-z0-9\-]{2,20}\]\]", fullmatch=True),
|
||||
min_size=0,
|
||||
max_size=3,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_artifact_metadata(draw: st.DrawFn) -> ArtifactMetadata:
|
||||
"""Generates a complete valid ArtifactMetadata object."""
|
||||
return ArtifactMetadata(
|
||||
type=draw(VALID_TYPES),
|
||||
title=draw(_safe_text(min_size=1, max_size=60)),
|
||||
tags=draw(st.lists(slugified_tag(), min_size=0, max_size=5, unique=True)),
|
||||
source_context=draw(source_context_strategy()),
|
||||
created=draw(optional_date()),
|
||||
updated=draw(optional_date()),
|
||||
shareable=draw(st.booleans()),
|
||||
links=draw(st.lists(artifact_link(), min_size=0, max_size=3)),
|
||||
content_hash=draw(content_hash_strategy()),
|
||||
source=draw(source_dict_strategy()),
|
||||
category=draw(
|
||||
st.sampled_from(
|
||||
["meeting", "decision", "project", "reference", "link", "inbox", ""]
|
||||
)
|
||||
),
|
||||
people=draw(people_strategy()),
|
||||
projects=draw(projects_strategy()),
|
||||
enrichment_pending=draw(st.booleans()),
|
||||
ocr_failed=draw(st.booleans()),
|
||||
fetch_failed=draw(st.booleans()),
|
||||
)
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_content(draw: st.DrawFn) -> str:
|
||||
"""Generates non-empty content for an artifact (valid Markdown body)."""
|
||||
# Simple lines of text that won't confuse YAML frontmatter detection
|
||||
lines = draw(
|
||||
st.lists(
|
||||
st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "P", "S", "Z"),
|
||||
blacklist_characters="\x00\r",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=80,
|
||||
).filter(lambda t: t.strip() != ""),
|
||||
min_size=1,
|
||||
max_size=5,
|
||||
)
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProperty1ArtifactSerializationRoundTrip:
|
||||
"""Property 1: Artifact Serialization Round-Trip.
|
||||
|
||||
**Validates: Requirements 16.6, 1.5**
|
||||
|
||||
For any valid ArtifactMetadata object with content, writing it to a Markdown
|
||||
file with YAML frontmatter and then reading it back SHALL produce an identical
|
||||
ArtifactMetadata object (fields, types, and values match).
|
||||
"""
|
||||
|
||||
@given(metadata=valid_artifact_metadata(), content=artifact_content())
|
||||
@settings(max_examples=200, deadline=5000)
|
||||
def test_roundtrip_preserves_all_metadata_fields(
|
||||
self, metadata: ArtifactMetadata, content: str
|
||||
) -> None:
|
||||
"""Writing frontmatter + content and parsing it back yields identical metadata."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp = Path(tmpdir)
|
||||
file_path = tmp / "test-artifact.md"
|
||||
|
||||
# Generate frontmatter and write to file
|
||||
frontmatter = _generate_frontmatter(metadata)
|
||||
file_content = frontmatter + content
|
||||
file_path.write_text(file_content, encoding="utf-8")
|
||||
|
||||
# Parse it back
|
||||
artifact = parse_frontmatter(file_path)
|
||||
parsed = artifact.metadata
|
||||
|
||||
# Assert all fields match
|
||||
assert parsed.type == metadata.type, (
|
||||
f"type mismatch: {parsed.type!r} != {metadata.type!r}"
|
||||
)
|
||||
assert parsed.title == metadata.title, (
|
||||
f"title mismatch: {parsed.title!r} != {metadata.title!r}"
|
||||
)
|
||||
assert parsed.tags == metadata.tags, (
|
||||
f"tags mismatch: {parsed.tags!r} != {metadata.tags!r}"
|
||||
)
|
||||
assert parsed.source_context == metadata.source_context, (
|
||||
f"source_context mismatch: {parsed.source_context!r} != {metadata.source_context!r}"
|
||||
)
|
||||
assert parsed.created == metadata.created, (
|
||||
f"created mismatch: {parsed.created!r} != {metadata.created!r}"
|
||||
)
|
||||
assert parsed.updated == metadata.updated, (
|
||||
f"updated mismatch: {parsed.updated!r} != {metadata.updated!r}"
|
||||
)
|
||||
assert parsed.shareable == metadata.shareable, (
|
||||
f"shareable mismatch: {parsed.shareable!r} != {metadata.shareable!r}"
|
||||
)
|
||||
assert parsed.content_hash == metadata.content_hash, (
|
||||
f"content_hash mismatch: {parsed.content_hash!r} != {metadata.content_hash!r}"
|
||||
)
|
||||
|
||||
# Links comparison (order matters)
|
||||
assert len(parsed.links) == len(metadata.links), (
|
||||
f"links length mismatch: {len(parsed.links)} != {len(metadata.links)}"
|
||||
)
|
||||
for i, (p_link, m_link) in enumerate(
|
||||
zip(parsed.links, metadata.links)
|
||||
):
|
||||
assert p_link.target == m_link.target, (
|
||||
f"links[{i}].target mismatch: {p_link.target!r} != {m_link.target!r}"
|
||||
)
|
||||
assert p_link.relation == m_link.relation, (
|
||||
f"links[{i}].relation mismatch: {p_link.relation!r} != {m_link.relation!r}"
|
||||
)
|
||||
|
||||
# Knowledge Management extensions
|
||||
assert parsed.source == metadata.source, (
|
||||
f"source mismatch: {parsed.source!r} != {metadata.source!r}"
|
||||
)
|
||||
assert parsed.category == metadata.category, (
|
||||
f"category mismatch: {parsed.category!r} != {metadata.category!r}"
|
||||
)
|
||||
assert parsed.people == metadata.people, (
|
||||
f"people mismatch: {parsed.people!r} != {metadata.people!r}"
|
||||
)
|
||||
assert parsed.projects == metadata.projects, (
|
||||
f"projects mismatch: {parsed.projects!r} != {metadata.projects!r}"
|
||||
)
|
||||
assert parsed.enrichment_pending == metadata.enrichment_pending, (
|
||||
f"enrichment_pending mismatch: {parsed.enrichment_pending!r} != {metadata.enrichment_pending!r}"
|
||||
)
|
||||
assert parsed.ocr_failed == metadata.ocr_failed, (
|
||||
f"ocr_failed mismatch: {parsed.ocr_failed!r} != {metadata.ocr_failed!r}"
|
||||
)
|
||||
assert parsed.fetch_failed == metadata.fetch_failed, (
|
||||
f"fetch_failed mismatch: {parsed.fetch_failed!r} != {metadata.fetch_failed!r}"
|
||||
)
|
||||
|
||||
# Also verify content is preserved
|
||||
assert artifact.content == content, (
|
||||
f"content mismatch: {artifact.content!r} != {content!r}"
|
||||
)
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Tests for QuickCapture – schnelle Wissenserfassung.
|
||||
|
||||
Requirements: 3.1, 3.2, 3.3, 3.7, 8.3
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def monorepo_root(tmp_path: Path) -> Path:
|
||||
"""Create a minimal monorepo structure for testing."""
|
||||
for ctx in ("bahn", "dhive", "privat"):
|
||||
(tmp_path / ctx / "knowledge" / "inbox").mkdir(parents=True)
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def capture(monorepo_root: Path) -> QuickCapture:
|
||||
"""Create a QuickCapture instance."""
|
||||
return QuickCapture(monorepo_root)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _detect_input_type tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDetectInputType:
|
||||
"""Tests for _detect_input_type."""
|
||||
|
||||
def test_detects_http_url(self, capture: QuickCapture) -> None:
|
||||
assert capture._detect_input_type("http://example.com") == "url"
|
||||
|
||||
def test_detects_https_url(self, capture: QuickCapture) -> None:
|
||||
assert capture._detect_input_type("https://docs.python.org/3/") == "url"
|
||||
|
||||
def test_detects_url_with_whitespace(self, capture: QuickCapture) -> None:
|
||||
assert capture._detect_input_type(" https://example.com ") == "url"
|
||||
|
||||
def test_detects_file_path_with_extension(self, capture: QuickCapture) -> None:
|
||||
assert capture._detect_input_type("/home/user/notes.md") == "file"
|
||||
|
||||
def test_detects_windows_path(self, capture: QuickCapture) -> None:
|
||||
assert capture._detect_input_type("C:\\Users\\doc.pdf") == "file"
|
||||
|
||||
def test_plain_text(self, capture: QuickCapture) -> None:
|
||||
assert capture._detect_input_type("Just a note about something") == "text"
|
||||
|
||||
def test_text_with_dot_but_no_separator(self, capture: QuickCapture) -> None:
|
||||
# e.g. "version 3.14" should not be detected as file
|
||||
assert capture._detect_input_type("version 3.14 is out") == "text"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _generate_filename tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGenerateFilename:
|
||||
"""Tests for _generate_filename."""
|
||||
|
||||
def test_filename_format(self, capture: QuickCapture) -> None:
|
||||
filename = capture._generate_filename("My Test Note")
|
||||
# Should match YYYY-MM-DD-HH-MM-slug.md
|
||||
pattern = r"^\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-my-test-note\.md$"
|
||||
assert re.match(pattern, filename), f"Unexpected filename: {filename}"
|
||||
|
||||
def test_filename_slugifies_special_chars(self, capture: QuickCapture) -> None:
|
||||
filename = capture._generate_filename("Hello, World! (2024)")
|
||||
assert "hello-world-2024" in filename
|
||||
|
||||
def test_filename_limits_slug_length(self, capture: QuickCapture) -> None:
|
||||
long_title = "a" * 100
|
||||
filename = capture._generate_filename(long_title)
|
||||
# Timestamp is 16 chars (YYYY-MM-DD-HH-MM) + 1 dash + slug (<=50) + .md
|
||||
slug_part = filename.split("-", 5)[-1].replace(".md", "")
|
||||
assert len(slug_part) <= 50
|
||||
|
||||
def test_filename_ends_with_md(self, capture: QuickCapture) -> None:
|
||||
filename = capture._generate_filename("test")
|
||||
assert filename.endswith(".md")
|
||||
|
||||
def test_filename_collapses_multiple_hyphens(self, capture: QuickCapture) -> None:
|
||||
filename = capture._generate_filename("foo---bar///baz")
|
||||
# Should not have consecutive hyphens in slug
|
||||
slug_part = filename[17:] # skip timestamp prefix "YYYY-MM-DD-HH-MM-"
|
||||
assert "--" not in slug_part
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# capture() integration tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCapture:
|
||||
"""Tests for capture()."""
|
||||
|
||||
def test_capture_text_creates_file(
|
||||
self, capture: QuickCapture, monorepo_root: Path
|
||||
) -> None:
|
||||
result = capture.capture("Eine schnelle Notiz", context="bahn")
|
||||
assert result.exists()
|
||||
assert result.parent == monorepo_root / "bahn" / "knowledge" / "inbox"
|
||||
|
||||
def test_capture_text_returns_path(self, capture: QuickCapture) -> None:
|
||||
result = capture.capture("Notiz", context="privat")
|
||||
assert isinstance(result, Path)
|
||||
|
||||
def test_capture_text_frontmatter(
|
||||
self, capture: QuickCapture, monorepo_root: Path
|
||||
) -> None:
|
||||
result = capture.capture(
|
||||
"Meeting notes from today",
|
||||
context="dhive",
|
||||
tags=["meeting", "project-x"],
|
||||
)
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "type: inbox" in content
|
||||
assert 'title: "Meeting notes from today"' in content
|
||||
assert "created:" in content
|
||||
assert "source:" in content
|
||||
assert "type: text" in content
|
||||
assert "tags: [meeting, project-x]" in content
|
||||
assert "source_context: dhive" in content
|
||||
|
||||
def test_capture_url_detected(
|
||||
self, capture: QuickCapture, monorepo_root: Path
|
||||
) -> None:
|
||||
url = "https://docs.python.org/3/library/pathlib.html"
|
||||
result = capture.capture(url, context="bahn")
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "type: url" in content
|
||||
assert f"[{url}]({url})" in content
|
||||
|
||||
def test_capture_file_detected(
|
||||
self, capture: QuickCapture, monorepo_root: Path
|
||||
) -> None:
|
||||
filepath = "/home/user/documents/report.pdf"
|
||||
result = capture.capture(filepath, context="privat")
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "type: file" in content
|
||||
assert f"Source file: `{filepath}`" in content
|
||||
|
||||
def test_capture_creates_inbox_dir_if_missing(
|
||||
self, capture: QuickCapture, monorepo_root: Path
|
||||
) -> None:
|
||||
# Remove inbox dir
|
||||
import shutil
|
||||
|
||||
inbox = monorepo_root / "bahn" / "knowledge" / "inbox"
|
||||
shutil.rmtree(inbox)
|
||||
assert not inbox.exists()
|
||||
|
||||
result = capture.capture("test", context="bahn")
|
||||
assert result.exists()
|
||||
assert inbox.exists()
|
||||
|
||||
def test_capture_empty_tags(
|
||||
self, capture: QuickCapture, monorepo_root: Path
|
||||
) -> None:
|
||||
result = capture.capture("note", context="bahn")
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "tags: []" in content
|
||||
|
||||
def test_capture_url_title_from_domain(
|
||||
self, capture: QuickCapture, monorepo_root: Path
|
||||
) -> None:
|
||||
result = capture.capture("https://example.com/docs/intro", context="bahn")
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "example.com/docs/intro" in content
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Property-Based Tests für Quick Capture.
|
||||
|
||||
**Validates: Requirements 3.3, 8.3**
|
||||
|
||||
Property 7: Filename Pattern from Title
|
||||
*For any* non-empty title string, the Quick Capture filename generator SHALL
|
||||
produce a filename matching the pattern
|
||||
`\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-[a-z0-9-]+\\.md`
|
||||
(date-time prefix + slugified title + .md extension).
|
||||
|
||||
Property 11: URL Detection in Quick Capture
|
||||
*For any* string that is a valid HTTP or HTTPS URL, the Quick Capture input
|
||||
type detector SHALL classify it as type "url". For any string that is not a
|
||||
valid HTTP(S) URL, it SHALL NOT classify it as "url".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings, assume
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared fixtures / helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FILENAME_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-[a-z0-9-]+\.md$")
|
||||
|
||||
|
||||
def make_capture() -> QuickCapture:
|
||||
"""Create a QuickCapture instance with a dummy root path."""
|
||||
return QuickCapture(Path("/tmp"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Non-empty title strings with various characters (letters, digits, punctuation, spaces).
|
||||
# Filtered to contain at least one alphanumeric char, since the slugifier strips
|
||||
# all non-alnum chars — a title like "/" would yield an empty slug. In practice,
|
||||
# _derive_title always produces titles with alphanumeric content (from URLs,
|
||||
# file stems, or first 60 chars of text).
|
||||
title_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "P", "Z"),
|
||||
blacklist_characters="\x00",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=200,
|
||||
).filter(lambda s: s.strip() != "" and any(c.isalnum() for c in s))
|
||||
|
||||
# Valid HTTP/HTTPS URLs: protocol + domain + optional path
|
||||
_domain_st = st.from_regex(r"[a-z][a-z0-9]{1,20}\.[a-z]{2,6}", fullmatch=True)
|
||||
_path_segment_st = st.from_regex(r"[a-z0-9\-]{1,15}", fullmatch=True)
|
||||
_url_path_st = st.lists(_path_segment_st, min_size=0, max_size=3).map(
|
||||
lambda parts: "/" + "/".join(parts) if parts else ""
|
||||
)
|
||||
_protocol_st = st.sampled_from(["http://", "https://"])
|
||||
|
||||
valid_url_st = st.builds(
|
||||
lambda proto, domain, path: f"{proto}{domain}{path}",
|
||||
_protocol_st,
|
||||
_domain_st,
|
||||
_url_path_st,
|
||||
)
|
||||
|
||||
# Non-URL strings: plain text that does NOT start with http:// or https://
|
||||
non_url_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "P", "Z"),
|
||||
blacklist_characters="\x00",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=100,
|
||||
).filter(
|
||||
lambda s: not s.strip().startswith("http://")
|
||||
and not s.strip().startswith("https://")
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 7: Filename Pattern from Title
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFilenamePatternFromTitle:
|
||||
"""**Validates: Requirements 3.3**
|
||||
|
||||
Property 7: For any non-empty title string, _generate_filename SHALL produce
|
||||
a filename matching the expected pattern with timestamp prefix, slugified
|
||||
title, and .md extension.
|
||||
"""
|
||||
|
||||
@given(title=title_st)
|
||||
@settings(max_examples=300)
|
||||
def test_filename_matches_pattern(self, title: str) -> None:
|
||||
"""Any non-empty title produces a filename matching the date-slug pattern."""
|
||||
capture = make_capture()
|
||||
filename = capture._generate_filename(title)
|
||||
assert FILENAME_PATTERN.match(filename), (
|
||||
f"Filename '{filename}' from title '{title!r}' "
|
||||
f"does not match pattern {FILENAME_PATTERN.pattern}"
|
||||
)
|
||||
|
||||
@given(title=title_st)
|
||||
@settings(max_examples=300)
|
||||
def test_slug_portion_max_50_chars(self, title: str) -> None:
|
||||
"""The slug portion of the filename never exceeds 50 characters."""
|
||||
capture = make_capture()
|
||||
filename = capture._generate_filename(title)
|
||||
# Filename format: YYYY-MM-DD-HH-MM-{slug}.md
|
||||
# Timestamp prefix is exactly "YYYY-MM-DD-HH-MM-" = 17 chars
|
||||
# Suffix is ".md" = 3 chars
|
||||
slug = filename[17:-3] # strip timestamp prefix and .md suffix
|
||||
assert len(slug) <= 50, (
|
||||
f"Slug '{slug}' has {len(slug)} chars, exceeds 50 limit"
|
||||
)
|
||||
|
||||
@given(title=title_st)
|
||||
@settings(max_examples=300)
|
||||
def test_filename_ends_with_md(self, title: str) -> None:
|
||||
"""Every generated filename ends with .md."""
|
||||
capture = make_capture()
|
||||
filename = capture._generate_filename(title)
|
||||
assert filename.endswith(".md")
|
||||
|
||||
@given(title=title_st)
|
||||
@settings(max_examples=200)
|
||||
def test_slug_has_no_trailing_hyphen(self, title: str) -> None:
|
||||
"""The slug portion does not end with a trailing hyphen."""
|
||||
capture = make_capture()
|
||||
filename = capture._generate_filename(title)
|
||||
# Remove .md suffix, then check the last char before .md
|
||||
without_md = filename[:-3]
|
||||
assert not without_md.endswith("-"), (
|
||||
f"Filename '{filename}' has trailing hyphen in slug"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 11: URL Detection in Quick Capture
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestURLDetectionInQuickCapture:
|
||||
"""**Validates: Requirements 8.3**
|
||||
|
||||
Property 11: For any valid HTTP(S) URL, _detect_input_type SHALL return "url".
|
||||
For any non-URL string, it SHALL NOT return "url".
|
||||
"""
|
||||
|
||||
@given(url=valid_url_st)
|
||||
@settings(max_examples=300)
|
||||
def test_valid_urls_classified_as_url(self, url: str) -> None:
|
||||
"""Any string starting with http:// or https:// followed by a domain is 'url'."""
|
||||
capture = make_capture()
|
||||
result = capture._detect_input_type(url)
|
||||
assert result == "url", (
|
||||
f"URL '{url}' was classified as '{result}', expected 'url'"
|
||||
)
|
||||
|
||||
@given(text=non_url_st)
|
||||
@settings(max_examples=300)
|
||||
def test_non_urls_not_classified_as_url(self, text: str) -> None:
|
||||
"""Any string that is NOT a valid HTTP(S) URL SHALL NOT be classified as 'url'."""
|
||||
capture = make_capture()
|
||||
result = capture._detect_input_type(text)
|
||||
assert result != "url", (
|
||||
f"Non-URL text '{text!r}' was incorrectly classified as 'url'"
|
||||
)
|
||||
|
||||
@given(url=valid_url_st)
|
||||
@settings(max_examples=200)
|
||||
def test_urls_with_leading_whitespace_still_detected(self, url: str) -> None:
|
||||
"""URLs with leading/trailing whitespace are still classified as 'url'."""
|
||||
capture = make_capture()
|
||||
padded = f" {url} "
|
||||
result = capture._detect_input_type(padded)
|
||||
assert result == "url", (
|
||||
f"Padded URL '{padded}' was classified as '{result}', expected 'url'"
|
||||
)
|
||||
@@ -0,0 +1,376 @@
|
||||
"""Unit-Tests für die ChatSource-Quellstrategie.
|
||||
|
||||
Testet Text- und JSON-Chat-Export-Parsing, Teilnehmer-Erkennung
|
||||
und Artefakt-Generierung.
|
||||
|
||||
Requirements: 7.3, 7.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
from monorepo.knowledge.sources.chat import (
|
||||
ChatSource,
|
||||
_parse_date_str,
|
||||
_slugify,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chat_dir(tmp_path: Path) -> Path:
|
||||
"""Creates a temporary directory for chat export files."""
|
||||
chat_dir = tmp_path / "chats"
|
||||
chat_dir.mkdir()
|
||||
return chat_dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_config(chat_dir: Path) -> SourceConfig:
|
||||
"""A SourceConfig pointing to the chat directory."""
|
||||
return SourceConfig(
|
||||
type="chat",
|
||||
name="Test Chats",
|
||||
params={"directory": str(chat_dir)},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def json_chat_content() -> str:
|
||||
"""Sample JSON chat export."""
|
||||
messages = [
|
||||
{
|
||||
"sender": "Alice",
|
||||
"timestamp": "2025-01-15 10:00",
|
||||
"text": "Good morning!",
|
||||
},
|
||||
{
|
||||
"sender": "Bob",
|
||||
"timestamp": "2025-01-15 10:05",
|
||||
"text": "Morning! Ready for the meeting?",
|
||||
},
|
||||
{
|
||||
"sender": "Alice",
|
||||
"timestamp": "2025-01-15 10:06",
|
||||
"text": "Yes, let's go.",
|
||||
},
|
||||
]
|
||||
return json.dumps(messages)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def text_chat_content() -> str:
|
||||
"""Sample text chat export with timestamp pattern."""
|
||||
return (
|
||||
"[2025-02-20 14:30] Max: Hallo zusammen\n"
|
||||
"[2025-02-20 14:31] Anna: Hi Max!\n"
|
||||
"[2025-02-20 14:32] Max: Wann ist das nächste Meeting?\n"
|
||||
"[2025-02-20 14:35] Anna: Morgen um 10 Uhr.\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def whatsapp_chat_content() -> str:
|
||||
"""Sample WhatsApp-style text chat export."""
|
||||
return (
|
||||
"2025-03-10, 09:00 - Alice: Good morning team\n"
|
||||
"2025-03-10, 09:01 - Bob: Morning!\n"
|
||||
"2025-03-10, 09:05 - Charlie: Hi all, let's discuss the sprint\n"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseDateStr:
|
||||
def test_iso_datetime(self) -> None:
|
||||
result = _parse_date_str("2025-01-15 10:30:00")
|
||||
assert result is not None
|
||||
assert result.year == 2025
|
||||
assert result.month == 1
|
||||
assert result.hour == 10
|
||||
|
||||
def test_iso_datetime_no_seconds(self) -> None:
|
||||
result = _parse_date_str("2025-01-15 10:30")
|
||||
assert result is not None
|
||||
assert result.minute == 30
|
||||
|
||||
def test_german_format(self) -> None:
|
||||
result = _parse_date_str("15.01.2025, 10:30")
|
||||
assert result is not None
|
||||
assert result.year == 2025
|
||||
assert result.day == 15
|
||||
|
||||
def test_invalid_returns_none(self) -> None:
|
||||
assert _parse_date_str("not a date") is None
|
||||
|
||||
def test_empty_returns_none(self) -> None:
|
||||
assert _parse_date_str("") is None
|
||||
|
||||
|
||||
class TestSlugify:
|
||||
def test_simple_text(self) -> None:
|
||||
assert _slugify("Team Chat") == "team-chat"
|
||||
|
||||
def test_special_chars(self) -> None:
|
||||
result = _slugify("chat_export (1)")
|
||||
assert "chat" in result
|
||||
assert "export" in result
|
||||
|
||||
def test_empty(self) -> None:
|
||||
assert _slugify("") == "chat-export"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: ChatSource.extract() with JSON
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestChatSourceJson:
|
||||
def test_extracts_json_chat(
|
||||
self, chat_dir: Path, source_config: SourceConfig, json_chat_content: str
|
||||
) -> None:
|
||||
"""Test JSON chat export parsing."""
|
||||
(chat_dir / "team-chat.json").write_text(json_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
assert len(result.errors) == 0
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.type == "chat"
|
||||
assert "Chat:" in artifact.metadata.title
|
||||
assert artifact.metadata.source["type"] == "chat"
|
||||
assert artifact.metadata.source["format"] == "json"
|
||||
|
||||
def test_json_participants_extracted(
|
||||
self, chat_dir: Path, source_config: SourceConfig, json_chat_content: str
|
||||
) -> None:
|
||||
"""Test that participants are extracted from JSON chat."""
|
||||
(chat_dir / "team-chat.json").write_text(json_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert "Alice" in artifact.metadata.people
|
||||
assert "Bob" in artifact.metadata.people
|
||||
|
||||
def test_json_date_range(
|
||||
self, chat_dir: Path, source_config: SourceConfig, json_chat_content: str
|
||||
) -> None:
|
||||
"""Test that date range is extracted from JSON chat."""
|
||||
(chat_dir / "team-chat.json").write_text(json_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.created is not None
|
||||
assert artifact.metadata.created.year == 2025
|
||||
assert artifact.metadata.created.month == 1
|
||||
|
||||
def test_json_messages_in_content(
|
||||
self, chat_dir: Path, source_config: SourceConfig, json_chat_content: str
|
||||
) -> None:
|
||||
"""Test that messages appear in artifact content."""
|
||||
(chat_dir / "team-chat.json").write_text(json_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert "Good morning!" in artifact.content
|
||||
assert "Alice" in artifact.content
|
||||
assert "Bob" in artifact.content
|
||||
|
||||
def test_empty_json_array_skipped(
|
||||
self, chat_dir: Path, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Test that empty JSON array produces no artifact."""
|
||||
(chat_dir / "empty.json").write_text("[]", encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
|
||||
def test_non_array_json_skipped(
|
||||
self, chat_dir: Path, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Test that non-array JSON is skipped."""
|
||||
(chat_dir / "object.json").write_text('{"key": "value"}', encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
|
||||
def test_json_alternative_keys(
|
||||
self, chat_dir: Path, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Test that alternative JSON keys (author, message, date) work."""
|
||||
messages = [
|
||||
{"author": "User1", "date": "2025-03-01 09:00", "message": "Hello"},
|
||||
{"from": "User2", "time": "2025-03-01 09:01", "content": "Hi there"},
|
||||
]
|
||||
(chat_dir / "alt.json").write_text(json.dumps(messages), encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert "Hello" in artifact.content
|
||||
assert "Hi there" in artifact.content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: ChatSource.extract() with Text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestChatSourceText:
|
||||
def test_extracts_text_chat_bracket_format(
|
||||
self, chat_dir: Path, source_config: SourceConfig, text_chat_content: str
|
||||
) -> None:
|
||||
"""Test text chat export with [timestamp] format."""
|
||||
(chat_dir / "team.txt").write_text(text_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.type == "chat"
|
||||
assert artifact.metadata.source["format"] == "text"
|
||||
|
||||
def test_text_chat_participants(
|
||||
self, chat_dir: Path, source_config: SourceConfig, text_chat_content: str
|
||||
) -> None:
|
||||
"""Test participant extraction from text chat."""
|
||||
(chat_dir / "team.txt").write_text(text_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert "Max" in artifact.metadata.people
|
||||
assert "Anna" in artifact.metadata.people
|
||||
|
||||
def test_whatsapp_format(
|
||||
self, chat_dir: Path, source_config: SourceConfig, whatsapp_chat_content: str
|
||||
) -> None:
|
||||
"""Test WhatsApp-style chat export parsing."""
|
||||
(chat_dir / "whatsapp.txt").write_text(whatsapp_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert "Alice" in artifact.metadata.people
|
||||
assert "Bob" in artifact.metadata.people
|
||||
assert "Charlie" in artifact.metadata.people
|
||||
|
||||
def test_unrecognized_format_still_creates_artifact(
|
||||
self, chat_dir: Path, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Test that unrecognized text format still creates an artifact."""
|
||||
(chat_dir / "freeform.txt").write_text(
|
||||
"Just some notes\nfrom a conversation\nwith no timestamps",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert "notes" in artifact.content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: ChatSource general behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestChatSourceGeneral:
|
||||
def test_empty_directory(self, chat_dir: Path, source_config: SourceConfig) -> None:
|
||||
"""Test that empty directory returns no artifacts."""
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 0
|
||||
|
||||
def test_nonexistent_directory(self, tmp_path: Path) -> None:
|
||||
"""Test error for non-existent directory."""
|
||||
config = SourceConfig(
|
||||
type="chat",
|
||||
name="Missing",
|
||||
params={"directory": str(tmp_path / "nonexistent")},
|
||||
)
|
||||
source = ChatSource()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 1
|
||||
assert "existiert nicht" in result.errors[0].message
|
||||
|
||||
def test_supports_incremental_returns_false(self) -> None:
|
||||
"""Chat source does not support incremental processing."""
|
||||
source = ChatSource()
|
||||
assert source.supports_incremental() is False
|
||||
|
||||
def test_content_hash_is_set(
|
||||
self, chat_dir: Path, source_config: SourceConfig, json_chat_content: str
|
||||
) -> None:
|
||||
"""Test that content hash is computed for chat artifacts."""
|
||||
(chat_dir / "chat.json").write_text(json_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.content_hash.startswith("sha256:")
|
||||
|
||||
def test_multiple_files_processed(
|
||||
self,
|
||||
chat_dir: Path,
|
||||
source_config: SourceConfig,
|
||||
json_chat_content: str,
|
||||
text_chat_content: str,
|
||||
) -> None:
|
||||
"""Test that multiple chat files in directory are all processed."""
|
||||
(chat_dir / "chat1.json").write_text(json_chat_content, encoding="utf-8")
|
||||
(chat_dir / "chat2.txt").write_text(text_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 2
|
||||
|
||||
def test_category_is_inbox(
|
||||
self, chat_dir: Path, source_config: SourceConfig, json_chat_content: str
|
||||
) -> None:
|
||||
"""Test that chat artifacts are categorized as inbox by default."""
|
||||
(chat_dir / "chat.json").write_text(json_chat_content, encoding="utf-8")
|
||||
|
||||
source = ChatSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert result.artifacts[0].metadata.category == "inbox"
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Property-Based Tests für Environment Variable Resolution in Config.
|
||||
|
||||
**Validates: Requirements 12.4, 12.6**
|
||||
|
||||
Property 21: Environment Variable Resolution in Config
|
||||
*For any* string containing ${VAR_NAME} patterns where the environment variable
|
||||
exists, the resolution function SHALL substitute the pattern with the variable's
|
||||
value. For any ${VAR_NAME} where the variable is NOT set, the validation SHALL
|
||||
report an error.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, assume
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.ingestion.config import (
|
||||
ConfigValidationError,
|
||||
resolve_env_vars,
|
||||
_resolve_env_vars_recursive,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Valid environment variable names: start with uppercase letter, then uppercase/digits/underscore
|
||||
# Use a prefix to avoid collisions with real env vars
|
||||
env_var_name_st = st.from_regex(r"_PBT[A-Z][A-Z0-9_]{0,20}", fullmatch=True)
|
||||
|
||||
# Values for env vars: non-empty text without null bytes
|
||||
env_var_value_st = st.text(
|
||||
alphabet=st.characters(blacklist_categories=("Cs",), blacklist_characters="\x00"),
|
||||
min_size=1,
|
||||
max_size=100,
|
||||
)
|
||||
|
||||
# Literal text segments (no ${...} patterns, no null bytes)
|
||||
literal_text_st = st.text(
|
||||
alphabet=st.characters(
|
||||
blacklist_categories=("Cs",),
|
||||
blacklist_characters="\x00$",
|
||||
),
|
||||
min_size=0,
|
||||
max_size=50,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property: Substitution Correctness (single variable)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnvVarSubstitutionProperty:
|
||||
"""**Validates: Requirements 12.4, 12.6**"""
|
||||
|
||||
@given(
|
||||
var_name=env_var_name_st,
|
||||
var_value=env_var_value_st,
|
||||
prefix=literal_text_st,
|
||||
suffix=literal_text_st,
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_single_var_substitution(
|
||||
self,
|
||||
var_name: str,
|
||||
var_value: str,
|
||||
prefix: str,
|
||||
suffix: str,
|
||||
) -> None:
|
||||
"""When an env var is set, resolve_env_vars substitutes ${VAR} with its value."""
|
||||
os.environ[var_name] = var_value
|
||||
try:
|
||||
input_str = f"{prefix}${{{var_name}}}{suffix}"
|
||||
result = resolve_env_vars(input_str)
|
||||
assert result == f"{prefix}{var_value}{suffix}"
|
||||
finally:
|
||||
os.environ.pop(var_name, None)
|
||||
|
||||
@given(
|
||||
var_names=st.lists(env_var_name_st, min_size=2, max_size=5, unique=True),
|
||||
var_values=st.lists(env_var_value_st, min_size=2, max_size=5),
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_multiple_vars_all_resolved(
|
||||
self,
|
||||
var_names: list[str],
|
||||
var_values: list[str],
|
||||
) -> None:
|
||||
"""All ${VAR_NAME} patterns in a string get resolved when all vars are set."""
|
||||
# Ensure we have matching counts
|
||||
count = min(len(var_names), len(var_values))
|
||||
var_names = var_names[:count]
|
||||
var_values = var_values[:count]
|
||||
|
||||
# Set all env vars
|
||||
for name, value in zip(var_names, var_values):
|
||||
os.environ[name] = value
|
||||
try:
|
||||
# Build input string with all vars separated by literal dashes
|
||||
input_str = "-".join(f"${{{name}}}" for name in var_names)
|
||||
result = resolve_env_vars(input_str)
|
||||
expected = "-".join(var_values)
|
||||
assert result == expected
|
||||
finally:
|
||||
for name in var_names:
|
||||
os.environ.pop(name, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property: Missing Variable Raises Error
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnvVarMissingRaisesError:
|
||||
"""**Validates: Requirements 12.4, 12.6**"""
|
||||
|
||||
@given(var_name=env_var_name_st)
|
||||
@settings(max_examples=100)
|
||||
def test_unset_var_raises_config_error(
|
||||
self,
|
||||
var_name: str,
|
||||
) -> None:
|
||||
"""When a referenced env var is NOT set, resolve_env_vars raises ConfigValidationError."""
|
||||
os.environ.pop(var_name, None)
|
||||
input_str = f"${{{var_name}}}"
|
||||
with pytest.raises(ConfigValidationError) as exc_info:
|
||||
resolve_env_vars(input_str)
|
||||
# The error message should mention the variable name
|
||||
assert var_name in str(exc_info.value)
|
||||
|
||||
@given(
|
||||
set_name=env_var_name_st,
|
||||
set_value=env_var_value_st,
|
||||
unset_name=env_var_name_st,
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_partial_resolution_raises_for_missing(
|
||||
self,
|
||||
set_name: str,
|
||||
set_value: str,
|
||||
unset_name: str,
|
||||
) -> None:
|
||||
"""When some vars exist and others don't, error is raised for missing ones."""
|
||||
assume(set_name != unset_name)
|
||||
os.environ[set_name] = set_value
|
||||
os.environ.pop(unset_name, None)
|
||||
try:
|
||||
input_str = f"${{{set_name}}}-${{{unset_name}}}"
|
||||
with pytest.raises(ConfigValidationError) as exc_info:
|
||||
resolve_env_vars(input_str)
|
||||
assert unset_name in str(exc_info.value)
|
||||
finally:
|
||||
os.environ.pop(set_name, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property: No Vars = Passthrough
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnvVarPassthrough:
|
||||
"""**Validates: Requirements 12.4, 12.6**"""
|
||||
|
||||
@given(text=literal_text_st)
|
||||
@settings(max_examples=100)
|
||||
def test_no_var_patterns_passthrough(self, text: str) -> None:
|
||||
"""Strings without ${...} patterns pass through unchanged."""
|
||||
# Ensure no ${...} pattern exists
|
||||
assume("${" not in text)
|
||||
result = resolve_env_vars(text)
|
||||
assert result == text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property: Recursive Resolution in Dicts/Lists
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnvVarRecursiveResolution:
|
||||
"""**Validates: Requirements 12.4, 12.6**"""
|
||||
|
||||
@given(
|
||||
var_name=env_var_name_st,
|
||||
var_value=env_var_value_st,
|
||||
key1=st.text(min_size=1, max_size=10, alphabet="abcdefghijklmnop"),
|
||||
key2=st.text(min_size=1, max_size=10, alphabet="abcdefghijklmnop"),
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_recursive_dict_resolution(
|
||||
self,
|
||||
var_name: str,
|
||||
var_value: str,
|
||||
key1: str,
|
||||
key2: str,
|
||||
) -> None:
|
||||
"""_resolve_env_vars_recursive handles nested dicts with ${VAR} patterns."""
|
||||
assume(key1 != key2)
|
||||
os.environ[var_name] = var_value
|
||||
try:
|
||||
nested = {
|
||||
key1: f"${{{var_name}}}",
|
||||
key2: {"inner": f"prefix-${{{var_name}}}-suffix"},
|
||||
}
|
||||
result = _resolve_env_vars_recursive(nested)
|
||||
assert result[key1] == var_value
|
||||
assert result[key2]["inner"] == f"prefix-{var_value}-suffix"
|
||||
finally:
|
||||
os.environ.pop(var_name, None)
|
||||
|
||||
@given(
|
||||
var_name=env_var_name_st,
|
||||
var_value=env_var_value_st,
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_recursive_list_resolution(
|
||||
self,
|
||||
var_name: str,
|
||||
var_value: str,
|
||||
) -> None:
|
||||
"""_resolve_env_vars_recursive handles lists with ${VAR} patterns."""
|
||||
os.environ[var_name] = var_value
|
||||
try:
|
||||
input_list = [f"${{{var_name}}}", "no_var", f"x-${{{var_name}}}-y"]
|
||||
result = _resolve_env_vars_recursive(input_list)
|
||||
assert result[0] == var_value
|
||||
assert result[1] == "no_var"
|
||||
assert result[2] == f"x-{var_value}-y"
|
||||
finally:
|
||||
os.environ.pop(var_name, None)
|
||||
|
||||
@given(
|
||||
int_val=st.integers(),
|
||||
float_val=st.floats(allow_nan=False, allow_infinity=False),
|
||||
bool_val=st.booleans(),
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_recursive_non_string_passthrough(
|
||||
self,
|
||||
int_val: int,
|
||||
float_val: float,
|
||||
bool_val: bool,
|
||||
) -> None:
|
||||
"""Non-string values pass through _resolve_env_vars_recursive unchanged."""
|
||||
assert _resolve_env_vars_recursive(int_val) == int_val
|
||||
assert _resolve_env_vars_recursive(float_val) == float_val
|
||||
assert _resolve_env_vars_recursive(bool_val) == bool_val
|
||||
assert _resolve_env_vars_recursive(None) is None
|
||||
@@ -0,0 +1,609 @@
|
||||
"""Unit-Tests für die Confluence-Quellstrategie.
|
||||
|
||||
Testet ConfluenceSource mit gemockten HTTP-Responses:
|
||||
- Einzelne Seiten abrufen
|
||||
- Seitenbäume traversieren
|
||||
- Space-Seiten abrufen
|
||||
- Inkrementelle Updates via Versions-Nummer
|
||||
- Fehlerbehandlung (API-Fehler, Parse-Fehler)
|
||||
- Frontmatter-Anreicherung (URL, Space, Titel, Last-Modified)
|
||||
- HTML-zu-Markdown-Konvertierung
|
||||
|
||||
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
from monorepo.knowledge.sources.confluence import (
|
||||
ConfluenceClient,
|
||||
ConfluencePage,
|
||||
ConfluenceSource,
|
||||
_html_to_markdown,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def confluence_source() -> ConfluenceSource:
|
||||
"""Erstellt eine ConfluenceSource-Instanz."""
|
||||
return ConfluenceSource()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_config() -> SourceConfig:
|
||||
"""Erstellt eine gültige SourceConfig für Confluence."""
|
||||
return SourceConfig(
|
||||
type="confluence",
|
||||
name="Test Confluence",
|
||||
params={
|
||||
"base_url": "https://confluence.example.com",
|
||||
"token": "test-api-token-123",
|
||||
"spaces": ["ACV2"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def page_config() -> SourceConfig:
|
||||
"""SourceConfig mit einzelnen Seiten-IDs."""
|
||||
return SourceConfig(
|
||||
type="confluence",
|
||||
name="Page Source",
|
||||
params={
|
||||
"base_url": "https://confluence.example.com",
|
||||
"api_key": "my-api-key",
|
||||
"pages": ["12345", "67890"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tree_config() -> SourceConfig:
|
||||
"""SourceConfig mit Seitenbäumen."""
|
||||
return SourceConfig(
|
||||
type="confluence",
|
||||
name="Tree Source",
|
||||
params={
|
||||
"base_url": "https://confluence.example.com",
|
||||
"token": "token-123",
|
||||
"tree_roots": ["99999"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _make_page_response(
|
||||
page_id: str = "12345",
|
||||
title: str = "Sprint Planning W24",
|
||||
space_key: str = "ACV2",
|
||||
version: int = 3,
|
||||
last_modified: str = "2025-01-15T10:30:00.000+01:00",
|
||||
content_html: str = "<p>Inhalt der Seite</p>",
|
||||
) -> dict:
|
||||
"""Erzeugt eine typische Confluence-API-Antwort für eine Seite."""
|
||||
return {
|
||||
"id": page_id,
|
||||
"title": title,
|
||||
"space": {"key": space_key},
|
||||
"version": {"number": version, "when": last_modified},
|
||||
"body": {"storage": {"value": content_html}},
|
||||
"_links": {"webui": f"/display/{space_key}/{title.replace(' ', '+')}"},
|
||||
"children": {
|
||||
"attachment": {
|
||||
"results": [
|
||||
{
|
||||
"title": "diagram.png",
|
||||
"_links": {"download": "/download/attachments/12345/diagram.png"},
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _make_child_response(child_ids: list[str]) -> dict:
|
||||
"""Erzeugt eine Confluence-API-Antwort für Kind-Seiten."""
|
||||
return {
|
||||
"results": [{"id": pid} for pid in child_ids],
|
||||
"size": len(child_ids),
|
||||
}
|
||||
|
||||
|
||||
def _make_space_response(page_ids: list[str], total: int | None = None) -> dict:
|
||||
"""Erzeugt eine Confluence-API-Antwort für Space-Seiten."""
|
||||
return {
|
||||
"results": [{"id": pid} for pid in page_ids],
|
||||
"size": total if total is not None else len(page_ids),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: HTML → Markdown Konvertierung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHtmlToMarkdown:
|
||||
"""Tests für die HTML-zu-Markdown-Konvertierung."""
|
||||
|
||||
def test_empty_html(self) -> None:
|
||||
assert _html_to_markdown("") == ""
|
||||
|
||||
def test_paragraph(self) -> None:
|
||||
result = _html_to_markdown("<p>Einfacher Text</p>")
|
||||
assert "Einfacher Text" in result
|
||||
|
||||
def test_headings(self) -> None:
|
||||
result = _html_to_markdown("<h1>Titel</h1><h2>Untertitel</h2>")
|
||||
assert "# Titel" in result
|
||||
assert "## Untertitel" in result
|
||||
|
||||
def test_bold_and_italic(self) -> None:
|
||||
result = _html_to_markdown("<strong>fett</strong> und <em>kursiv</em>")
|
||||
assert "**fett**" in result
|
||||
assert "*kursiv*" in result
|
||||
|
||||
def test_links(self) -> None:
|
||||
result = _html_to_markdown('<a href="https://example.com">Link</a>')
|
||||
assert "[Link](https://example.com)" in result
|
||||
|
||||
def test_unordered_list(self) -> None:
|
||||
result = _html_to_markdown("<ul><li>Eins</li><li>Zwei</li></ul>")
|
||||
assert "- Eins" in result
|
||||
assert "- Zwei" in result
|
||||
|
||||
def test_code_inline(self) -> None:
|
||||
result = _html_to_markdown("Nutze <code>git pull</code> bitte")
|
||||
assert "`git pull`" in result
|
||||
|
||||
def test_line_breaks(self) -> None:
|
||||
result = _html_to_markdown("Zeile 1<br/>Zeile 2")
|
||||
assert "Zeile 1\nZeile 2" in result
|
||||
|
||||
def test_html_entities_decoded(self) -> None:
|
||||
result = _html_to_markdown("<p>A & B < C</p>")
|
||||
assert "A & B < C" in result
|
||||
|
||||
def test_confluence_macros_removed(self) -> None:
|
||||
html = (
|
||||
'<ac:structured-macro ac:name="info">'
|
||||
"<ac:rich-text-body>Info text</ac:rich-text-body>"
|
||||
"</ac:structured-macro>"
|
||||
)
|
||||
result = _html_to_markdown(html)
|
||||
assert "ac:" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: ConfluenceSource Interface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConfluenceSourceInterface:
|
||||
"""Tests für das SourceStrategy-Interface."""
|
||||
|
||||
def test_supports_incremental(self, confluence_source: ConfluenceSource) -> None:
|
||||
assert confluence_source.supports_incremental() is True
|
||||
|
||||
def test_missing_base_url(self, confluence_source: ConfluenceSource) -> None:
|
||||
"""Fehlende base_url erzeugt einen Fehler."""
|
||||
config = SourceConfig(
|
||||
type="confluence",
|
||||
name="Bad Config",
|
||||
params={"token": "abc"},
|
||||
)
|
||||
result = confluence_source.extract(config, "bahn")
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "auth"
|
||||
assert "base_url" in result.errors[0].message
|
||||
|
||||
def test_missing_token(self, confluence_source: ConfluenceSource) -> None:
|
||||
"""Fehlender Token erzeugt einen Fehler."""
|
||||
config = SourceConfig(
|
||||
type="confluence",
|
||||
name="No Token",
|
||||
params={"base_url": "https://confluence.example.com"},
|
||||
)
|
||||
result = confluence_source.extract(config, "bahn")
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "auth"
|
||||
assert "Token" in result.errors[0].message
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Einzelne Seiten abrufen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPageExtraction:
|
||||
"""Tests für das Abrufen einzelner Seiten."""
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_single_page_extraction(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Einzelne Seite wird korrekt extrahiert."""
|
||||
mock_request.return_value = _make_page_response(
|
||||
page_id="12345",
|
||||
title="Sprint Planning W24",
|
||||
content_html="<h2>Agenda</h2><p>Punkt 1</p>",
|
||||
)
|
||||
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 2 # Two pages configured
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.title == "Sprint Planning W24"
|
||||
assert artifact.metadata.source["type"] == "confluence"
|
||||
assert artifact.metadata.source["space"] == "ACV2"
|
||||
assert "## Agenda" in artifact.content
|
||||
assert "Punkt 1" in artifact.content
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_frontmatter_enrichment(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Frontmatter wird mit URL, Space, Titel und Last-Modified angereichert."""
|
||||
mock_request.return_value = _make_page_response(
|
||||
page_id="12345",
|
||||
title="API Redesign",
|
||||
space_key="EINFACHBAHN",
|
||||
last_modified="2025-03-20T14:00:00.000+01:00",
|
||||
)
|
||||
|
||||
# Nur eine Seite konfigurieren
|
||||
page_config.params["pages"] = ["12345"]
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.source["url"].startswith("https://confluence.example.com")
|
||||
assert artifact.metadata.source["space"] == "EINFACHBAHN"
|
||||
assert artifact.metadata.source["page_id"] == "12345"
|
||||
assert artifact.metadata.source["version"] == "3"
|
||||
assert artifact.metadata.created == date(2025, 3, 20)
|
||||
assert artifact.metadata.source_context == "bahn"
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_content_hash_computed(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Content-Hash wird berechnet."""
|
||||
mock_request.return_value = _make_page_response(page_id="12345")
|
||||
page_config.params["pages"] = ["12345"]
|
||||
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.content_hash.startswith("sha256:")
|
||||
assert len(artifact.metadata.content_hash) == 71 # sha256: + 64 hex chars
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_attachments_listed(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Anhänge werden als Links im Artefakt vermerkt."""
|
||||
mock_request.return_value = _make_page_response(page_id="12345")
|
||||
page_config.params["pages"] = ["12345"]
|
||||
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert "## Anhänge" in artifact.content
|
||||
assert "diagram.png" in artifact.content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Inkrementelle Updates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIncrementalUpdates:
|
||||
"""Tests für inkrementelle Verarbeitung via Versions-Nummer."""
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_skip_unchanged_page(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Unveränderte Seiten werden übersprungen."""
|
||||
mock_request.return_value = _make_page_response(
|
||||
page_id="12345", version=3
|
||||
)
|
||||
page_config.params["pages"] = ["12345"]
|
||||
|
||||
# Version 3 ist bereits bekannt
|
||||
confluence_source.set_known_versions({"12345": 3})
|
||||
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert result.skipped == 1
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_process_updated_page(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Aktualisierte Seiten werden verarbeitet."""
|
||||
mock_request.return_value = _make_page_response(
|
||||
page_id="12345", version=5
|
||||
)
|
||||
page_config.params["pages"] = ["12345"]
|
||||
|
||||
# Version 3 ist bekannt, aber Seite ist jetzt bei Version 5
|
||||
confluence_source.set_known_versions({"12345": 3})
|
||||
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
assert result.skipped == 0
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_new_page_always_processed(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Neue Seiten (nicht in known_versions) werden immer verarbeitet."""
|
||||
mock_request.return_value = _make_page_response(
|
||||
page_id="12345", version=1
|
||||
)
|
||||
page_config.params["pages"] = ["12345"]
|
||||
|
||||
# Andere Seite ist bekannt, aber nicht diese
|
||||
confluence_source.set_known_versions({"99999": 10})
|
||||
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Seitenbäume
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPageTreeTraversal:
|
||||
"""Tests für die Seitenbaum-Traversierung."""
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_tree_traversal(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
tree_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Seitenbaum wird rekursiv traversiert."""
|
||||
# Request-Routing basierend auf URL-Pfad
|
||||
def side_effect(path: str) -> dict:
|
||||
if "/child/page" in path:
|
||||
if "99999" in path:
|
||||
return _make_child_response(["11111", "22222"])
|
||||
return _make_child_response([])
|
||||
# get_page calls
|
||||
if "99999" in path:
|
||||
return _make_page_response(page_id="99999", title="Root Page")
|
||||
if "11111" in path:
|
||||
return _make_page_response(page_id="11111", title="Child 1")
|
||||
if "22222" in path:
|
||||
return _make_page_response(page_id="22222", title="Child 2")
|
||||
return _make_page_response()
|
||||
|
||||
mock_request.side_effect = side_effect
|
||||
|
||||
result = confluence_source.extract(tree_config, "bahn")
|
||||
|
||||
# Root + 2 Children = 3 Seiten
|
||||
assert len(result.artifacts) == 3
|
||||
titles = {a.metadata.title for a in result.artifacts}
|
||||
assert "Root Page" in titles
|
||||
assert "Child 1" in titles
|
||||
assert "Child 2" in titles
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Space-Seiten
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSpaceExtraction:
|
||||
"""Tests für das Abrufen von Space-Seiten."""
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_space_pages_extracted(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
source_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Alle Seiten eines Spaces werden abgerufen."""
|
||||
def side_effect(path: str) -> dict:
|
||||
if "spaceKey=ACV2" in path:
|
||||
return _make_space_response(["111", "222", "333"])
|
||||
# Individual page requests
|
||||
page_id = path.split("/content/")[1].split("?")[0]
|
||||
return _make_page_response(page_id=page_id, title=f"Page {page_id}")
|
||||
|
||||
mock_request.side_effect = side_effect
|
||||
|
||||
result = confluence_source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Fehlerbehandlung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestErrorHandling:
|
||||
"""Tests für graceful Error Handling."""
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_api_error_skips_page(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""API-Fehler bei einzelner Seite überspringt diese und verarbeitet Rest."""
|
||||
call_count = [0]
|
||||
|
||||
def side_effect(path: str) -> dict:
|
||||
call_count[0] += 1
|
||||
if "12345" in path:
|
||||
raise urllib.error.HTTPError(
|
||||
url="https://confluence.example.com/rest/api/content/12345",
|
||||
code=404,
|
||||
msg="Not Found",
|
||||
hdrs={}, # type: ignore
|
||||
fp=None,
|
||||
)
|
||||
return _make_page_response(page_id="67890", title="Good Page")
|
||||
|
||||
mock_request.side_effect = side_effect
|
||||
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
# Eine Seite erfolgreich, eine fehlgeschlagen
|
||||
assert len(result.artifacts) == 1
|
||||
assert result.artifacts[0].metadata.title == "Good Page"
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "connection"
|
||||
assert "12345" in result.errors[0].message
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_space_api_error_logged(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
source_config: SourceConfig,
|
||||
) -> None:
|
||||
"""API-Fehler beim Space-Abruf wird geloggt, Pipeline fährt fort."""
|
||||
mock_request.side_effect = urllib.error.URLError("Connection refused")
|
||||
|
||||
result = confluence_source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "connection"
|
||||
assert result.errors[0].retry is True
|
||||
|
||||
@patch.object(ConfluenceClient, "_request")
|
||||
def test_parse_error_skips_page(
|
||||
self,
|
||||
mock_request: MagicMock,
|
||||
confluence_source: ConfluenceSource,
|
||||
page_config: SourceConfig,
|
||||
) -> None:
|
||||
"""Parse-Fehler wird als Fehler geloggt, Seite übersprungen."""
|
||||
# Ungültiges JSON-ähnliches Response (fehlende Felder)
|
||||
page_config.params["pages"] = ["12345"]
|
||||
mock_request.return_value = {"id": "12345"} # Missing required fields
|
||||
|
||||
result = confluence_source.extract(page_config, "bahn")
|
||||
|
||||
# Should still produce an artifact (with defaults) or error
|
||||
# The _parse_page_response handles missing fields gracefully
|
||||
assert len(result.artifacts) == 1 or len(result.errors) == 1
|
||||
|
||||
def test_no_pages_configured(self, confluence_source: ConfluenceSource) -> None:
|
||||
"""Keine Seiten konfiguriert → leeres Ergebnis ohne Fehler."""
|
||||
config = SourceConfig(
|
||||
type="confluence",
|
||||
name="Empty Config",
|
||||
params={
|
||||
"base_url": "https://confluence.example.com",
|
||||
"token": "abc",
|
||||
},
|
||||
)
|
||||
result = confluence_source.extract(config, "bahn")
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Confluence Client
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConfluenceClient:
|
||||
"""Tests für den ConfluenceClient."""
|
||||
|
||||
def test_bearer_auth_header(self) -> None:
|
||||
"""Bearer-Token wird korrekt gesetzt (Server/DC)."""
|
||||
client = ConfluenceClient(
|
||||
base_url="https://conf.example.com",
|
||||
token="my-token",
|
||||
)
|
||||
assert client._build_auth_header() == "Bearer my-token"
|
||||
|
||||
def test_basic_auth_header(self) -> None:
|
||||
"""Basic Auth wird korrekt gesetzt (Cloud)."""
|
||||
client = ConfluenceClient(
|
||||
base_url="https://conf.example.com",
|
||||
token="api-token",
|
||||
username="user@example.com",
|
||||
)
|
||||
header = client._build_auth_header()
|
||||
assert header.startswith("Basic ")
|
||||
|
||||
def test_base_url_trailing_slash_removed(self) -> None:
|
||||
"""Trailing Slash wird von base_url entfernt."""
|
||||
client = ConfluenceClient(
|
||||
base_url="https://conf.example.com/",
|
||||
token="token",
|
||||
)
|
||||
assert client.base_url == "https://conf.example.com"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Slugify
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSlugify:
|
||||
"""Tests für die Slug-Generierung."""
|
||||
|
||||
def test_simple_title(self) -> None:
|
||||
assert ConfluenceSource._slugify("Sprint Planning") == "sprint-planning"
|
||||
|
||||
def test_umlauts(self) -> None:
|
||||
assert ConfluenceSource._slugify("Über Änderungen") == "ueber-aenderungen"
|
||||
|
||||
def test_special_chars(self) -> None:
|
||||
assert ConfluenceSource._slugify("API (v2) - Redesign!") == "api-v2-redesign"
|
||||
|
||||
def test_empty_string(self) -> None:
|
||||
assert ConfluenceSource._slugify("") == "untitled"
|
||||
|
||||
def test_only_special_chars(self) -> None:
|
||||
assert ConfluenceSource._slugify("---") == "untitled"
|
||||
@@ -784,6 +784,552 @@ class TestSharedMirrorReadOnly:
|
||||
assert len(mirror_result.mirrored_paths) >= 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 8: Full Knowledge Pipeline E2E
|
||||
# Requirements: 2.2, 2.3, 5.1, 6.1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestKnowledgePipelineE2E:
|
||||
"""E2E: Full ingestion pipeline with source configs, enrichment, and indexing."""
|
||||
|
||||
@pytest.fixture()
|
||||
def knowledge_root(self, tmp_path: Path) -> Path:
|
||||
"""Creates a knowledge-ready monorepo structure."""
|
||||
root = tmp_path / "monorepo"
|
||||
root.mkdir()
|
||||
|
||||
# Context knowledge folders
|
||||
for ctx in ("bahn", "dhive", "privat"):
|
||||
kp = root / ctx / "knowledge"
|
||||
for sub in ("inbox", "meetings", "decisions", "projects",
|
||||
"people", "references", "links"):
|
||||
(kp / sub).mkdir(parents=True)
|
||||
# Empty YAML index
|
||||
(kp / "_index.yaml").write_text(
|
||||
"version: '1.0'\nlast_updated: ''\nartifacts: []\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return root
|
||||
|
||||
def test_full_pipeline_with_markdown_source(self, knowledge_root: Path) -> None:
|
||||
"""Full pipeline run: sources.yaml → MarkdownSource → enrich → store → index."""
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
from monorepo.knowledge.ingestion.enrichment import EnrichmentAgent, EnrichmentResult
|
||||
|
||||
ctx = "bahn"
|
||||
kp = knowledge_root / ctx / "knowledge"
|
||||
|
||||
# Create a markdown file source
|
||||
source_dir = knowledge_root / ctx / "docs"
|
||||
source_dir.mkdir(parents=True)
|
||||
(source_dir / "api-design.md").write_text(
|
||||
"---\n"
|
||||
"type: decision\n"
|
||||
"title: API Design Principles\n"
|
||||
"tags: [api, design]\n"
|
||||
"---\n\n"
|
||||
"# API Design\n\nWe use RESTful conventions.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Create sources.yaml
|
||||
sources_yaml = {
|
||||
"sources": [
|
||||
{
|
||||
"type": "markdown",
|
||||
"name": "bahn-docs",
|
||||
"enabled": True,
|
||||
"params": {"directory": str(source_dir)},
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
"create_tasks": False,
|
||||
"git": {"auto_commit": False},
|
||||
},
|
||||
}
|
||||
(kp / "sources.yaml").write_text(
|
||||
yaml.dump(sources_yaml), encoding="utf-8"
|
||||
)
|
||||
|
||||
# Mock enrichment to avoid LLM calls
|
||||
mock_enrichment = MagicMock(spec=EnrichmentAgent)
|
||||
mock_enrichment.enrich.return_value = EnrichmentResult(
|
||||
title="API Design Principles",
|
||||
category="decision",
|
||||
tags=["api", "design"],
|
||||
people=[],
|
||||
projects=[],
|
||||
action_items=[],
|
||||
confidence=0.9,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=knowledge_root,
|
||||
context=ctx,
|
||||
no_commit=True,
|
||||
enrichment_agent=mock_enrichment,
|
||||
)
|
||||
|
||||
result = pipeline.run()
|
||||
|
||||
# Assertions
|
||||
assert result.processed >= 1
|
||||
assert result.context == ctx
|
||||
assert len(result.errors) == 0
|
||||
|
||||
# Index should be updated
|
||||
index_content = (kp / "_index.yaml").read_text(encoding="utf-8")
|
||||
assert "api-design" in index_content or "API" in index_content
|
||||
|
||||
def test_quick_capture_to_inbox_to_ingest_categorization(
|
||||
self, knowledge_root: Path
|
||||
) -> None:
|
||||
"""Quick Capture → Inbox → pipeline.process_inbox() → categorization."""
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
from monorepo.knowledge.ingestion.enrichment import EnrichmentAgent, EnrichmentResult
|
||||
|
||||
ctx = "dhive"
|
||||
kp = knowledge_root / ctx / "knowledge"
|
||||
|
||||
# Step 1: Quick Capture a note
|
||||
capture = QuickCapture(monorepo_root=knowledge_root)
|
||||
captured_path = capture.capture(
|
||||
input_text="Meeting mit Team: Sprint Planning nächste Woche vorbereiten",
|
||||
context=ctx,
|
||||
tags=["sprint", "planning"],
|
||||
)
|
||||
|
||||
# Verify capture created file in inbox
|
||||
assert captured_path.exists()
|
||||
assert "inbox" in str(captured_path)
|
||||
content = captured_path.read_text(encoding="utf-8")
|
||||
assert "type: inbox" in content
|
||||
assert "sprint" in content
|
||||
|
||||
# Step 2: Process inbox (enrichment categorizes as meeting)
|
||||
mock_enrichment = MagicMock(spec=EnrichmentAgent)
|
||||
mock_enrichment.enrich.return_value = EnrichmentResult(
|
||||
title="Sprint Planning Vorbereitung",
|
||||
category="meeting",
|
||||
tags=["sprint", "planning", "team"],
|
||||
people=["Team"],
|
||||
projects=[],
|
||||
action_items=[],
|
||||
confidence=0.85,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=knowledge_root,
|
||||
context=ctx,
|
||||
no_commit=True,
|
||||
enrichment_agent=mock_enrichment,
|
||||
)
|
||||
|
||||
result = pipeline.process_inbox()
|
||||
|
||||
# Step 3: Verify categorization
|
||||
assert result.processed >= 1
|
||||
assert result.context == ctx
|
||||
|
||||
# Index should contain the processed artifact
|
||||
index_content = (kp / "_index.yaml").read_text(encoding="utf-8")
|
||||
assert len(index_content) > 50 # non-empty index
|
||||
|
||||
def test_confluence_source_with_mocked_api(self, knowledge_root: Path) -> None:
|
||||
"""Confluence source extraction with mocked REST API responses."""
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource, ConfluencePage
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
|
||||
source = ConfluenceSource()
|
||||
config = SourceConfig(
|
||||
type="confluence",
|
||||
name="bahn-confluence",
|
||||
params={
|
||||
"base_url": "https://confluence.bahn.de",
|
||||
"token": "fake-token",
|
||||
"pages": ["12345"],
|
||||
},
|
||||
)
|
||||
|
||||
# Mock the HTTP client interactions
|
||||
mock_page = ConfluencePage(
|
||||
page_id="12345",
|
||||
title="Architecture Decision Records",
|
||||
space_key="TEAM",
|
||||
version=3,
|
||||
content_html="<h1>ADR-001</h1><p>We use microservices architecture.</p>",
|
||||
url="https://confluence.bahn.de/pages/viewpage.action?pageId=12345",
|
||||
last_modified="2024-03-15T10:30:00Z",
|
||||
attachments=[],
|
||||
)
|
||||
|
||||
with patch.object(source, "_create_client") as mock_client_factory:
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_page.return_value = mock_page
|
||||
mock_client_factory.return_value = mock_client
|
||||
|
||||
with patch.object(source, "_collect_page_ids", return_value=["12345"]):
|
||||
extraction = source.extract(config, context="bahn")
|
||||
|
||||
# Verify extraction result
|
||||
assert len(extraction.artifacts) == 1
|
||||
assert len(extraction.errors) == 0
|
||||
|
||||
artifact = extraction.artifacts[0]
|
||||
assert "Architecture Decision Records" in artifact.metadata.title
|
||||
assert artifact.metadata.source.get("type") == "confluence"
|
||||
assert "confluence.bahn.de" in artifact.metadata.source.get("url", "")
|
||||
|
||||
def test_confluence_source_api_error_graceful(self, knowledge_root: Path) -> None:
|
||||
"""Confluence source handles API errors gracefully without crashing."""
|
||||
import urllib.error
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
|
||||
source = ConfluenceSource()
|
||||
config = SourceConfig(
|
||||
type="confluence",
|
||||
name="bahn-confluence",
|
||||
params={
|
||||
"base_url": "https://confluence.bahn.de",
|
||||
"token": "fake-token",
|
||||
"pages": ["99999"],
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(source, "_create_client") as mock_client_factory:
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_page.side_effect = urllib.error.URLError("Connection refused")
|
||||
mock_client_factory.return_value = mock_client
|
||||
|
||||
with patch.object(source, "_collect_page_ids", return_value=["99999"]):
|
||||
extraction = source.extract(config, context="bahn")
|
||||
|
||||
# Should not crash, should record error
|
||||
assert len(extraction.artifacts) == 0
|
||||
assert len(extraction.errors) >= 1
|
||||
assert extraction.errors[0].error_type == "connection"
|
||||
|
||||
def test_jira_source_with_mocked_api(self, knowledge_root: Path) -> None:
|
||||
"""Jira source extraction with mocked REST API responses."""
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
|
||||
source = JiraSource()
|
||||
config = SourceConfig(
|
||||
type="jira",
|
||||
name="bahn-jira",
|
||||
params={
|
||||
"base_url": "https://jira.bahn.de",
|
||||
"token": "fake-token",
|
||||
"username": "test-user",
|
||||
"jql": "project = ACV2 AND updatedDate > -7d",
|
||||
},
|
||||
)
|
||||
|
||||
# Mock issue data as Jira REST API v2 format
|
||||
mock_issues = [
|
||||
{
|
||||
"key": "ACV2-101",
|
||||
"fields": {
|
||||
"summary": "Implement OAuth2 login flow",
|
||||
"description": "Implement the OAuth2 flow for SSO integration.",
|
||||
"status": {"name": "In Progress"},
|
||||
"assignee": {"displayName": "Max Mustermann"},
|
||||
"reporter": {"displayName": "Anna Schmidt"},
|
||||
"project": {"key": "ACV2"},
|
||||
"comment": {"comments": [
|
||||
{"body": "Started working on this", "author": {"displayName": "Max Mustermann"}},
|
||||
]},
|
||||
"updated": "2024-03-20T14:00:00.000+0000",
|
||||
"created": "2024-03-18T09:00:00.000+0000",
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "ACV2-102",
|
||||
"fields": {
|
||||
"summary": "Fix logout redirect",
|
||||
"description": "Logout should redirect to landing page.",
|
||||
"status": {"name": "Done"},
|
||||
"assignee": {"displayName": "Anna Schmidt"},
|
||||
"reporter": {"displayName": "Max Mustermann"},
|
||||
"project": {"key": "ACV2"},
|
||||
"comment": {"comments": []},
|
||||
"updated": "2024-03-19T16:30:00.000+0000",
|
||||
"created": "2024-03-17T11:00:00.000+0000",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
with patch.object(source, "_fetch_issues", return_value=mock_issues):
|
||||
with patch.object(source, "_create_session", return_value=MagicMock()):
|
||||
extraction = source.extract(config, context="bahn")
|
||||
|
||||
# Verify extraction result
|
||||
assert len(extraction.artifacts) == 2
|
||||
assert len(extraction.errors) == 0
|
||||
|
||||
# Verify first artifact metadata
|
||||
art1 = extraction.artifacts[0]
|
||||
assert "ACV2-101" in art1.metadata.source.get("issue_key", "")
|
||||
assert art1.metadata.source.get("type") == "jira"
|
||||
assert "OAuth2" in art1.metadata.title or "OAuth2" in art1.content
|
||||
|
||||
def test_jira_source_api_error_graceful(self, knowledge_root: Path) -> None:
|
||||
"""Jira source handles API errors gracefully without crashing."""
|
||||
from monorepo.knowledge.sources.jira import JiraSource, JiraAPIError
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
|
||||
source = JiraSource()
|
||||
config = SourceConfig(
|
||||
type="jira",
|
||||
name="bahn-jira",
|
||||
params={
|
||||
"base_url": "https://jira.bahn.de",
|
||||
"token": "fake-token",
|
||||
"jql": "project = ACV2",
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
source, "_fetch_issues",
|
||||
side_effect=JiraAPIError("Connection timeout", error_type="connection", retry=True),
|
||||
):
|
||||
with patch.object(source, "_create_session", return_value=MagicMock()):
|
||||
extraction = source.extract(config, context="bahn")
|
||||
|
||||
# Should not crash, should record error
|
||||
assert len(extraction.artifacts) == 0
|
||||
assert len(extraction.errors) >= 1
|
||||
assert extraction.errors[0].error_type == "connection"
|
||||
assert extraction.errors[0].retry is True
|
||||
|
||||
def test_pipeline_with_confluence_and_jira_sources(self, knowledge_root: Path) -> None:
|
||||
"""Full pipeline E2E with both Confluence and Jira sources configured."""
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
from monorepo.knowledge.ingestion.enrichment import EnrichmentAgent, EnrichmentResult
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource, ConfluencePage
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.base import SourceStrategy
|
||||
|
||||
ctx = "bahn"
|
||||
kp = knowledge_root / ctx / "knowledge"
|
||||
|
||||
# Create sources.yaml with both sources
|
||||
sources_yaml = {
|
||||
"sources": [
|
||||
{
|
||||
"type": "confluence",
|
||||
"name": "bahn-confluence",
|
||||
"enabled": True,
|
||||
"params": {
|
||||
"base_url": "https://confluence.bahn.de",
|
||||
"token": "fake-token",
|
||||
"pages": ["100"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "jira",
|
||||
"name": "bahn-jira",
|
||||
"enabled": True,
|
||||
"params": {
|
||||
"base_url": "https://jira.bahn.de",
|
||||
"token": "fake-token",
|
||||
"username": "test",
|
||||
"jql": "project = ACV2",
|
||||
},
|
||||
},
|
||||
],
|
||||
"settings": {
|
||||
"create_tasks": False,
|
||||
"git": {"auto_commit": False},
|
||||
},
|
||||
}
|
||||
(kp / "sources.yaml").write_text(
|
||||
yaml.dump(sources_yaml), encoding="utf-8"
|
||||
)
|
||||
|
||||
# Mock enrichment
|
||||
mock_enrichment = MagicMock(spec=EnrichmentAgent)
|
||||
mock_enrichment.enrich.return_value = EnrichmentResult(
|
||||
title="Test Artifact",
|
||||
category="reference",
|
||||
tags=["test"],
|
||||
people=[],
|
||||
projects=[],
|
||||
action_items=[],
|
||||
confidence=0.8,
|
||||
)
|
||||
|
||||
# Create mock strategies that return pre-built artifacts
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import ExtractionResult
|
||||
from datetime import date
|
||||
|
||||
mock_confluence = MagicMock(spec=ConfluenceSource)
|
||||
mock_confluence.extract.return_value = ExtractionResult(
|
||||
artifacts=[
|
||||
Artifact(
|
||||
metadata=ArtifactMetadata(
|
||||
type="reference",
|
||||
title="Confluence Page: Architecture",
|
||||
tags=["architecture"],
|
||||
source_context=ctx,
|
||||
source={"type": "confluence", "url": "https://confluence.bahn.de/page/100"},
|
||||
created=date(2024, 3, 15),
|
||||
),
|
||||
content="# Architecture\n\nMicroservices approach.\n",
|
||||
file_path=Path("confluence-architecture.md"),
|
||||
),
|
||||
],
|
||||
errors=[],
|
||||
skipped=0,
|
||||
)
|
||||
|
||||
mock_jira = MagicMock(spec=JiraSource)
|
||||
mock_jira.extract.return_value = ExtractionResult(
|
||||
artifacts=[
|
||||
Artifact(
|
||||
metadata=ArtifactMetadata(
|
||||
type="reference",
|
||||
title="ACV2-200: Feature Request",
|
||||
tags=["jira", "feature"],
|
||||
source_context=ctx,
|
||||
source={"type": "jira", "issue_key": "ACV2-200"},
|
||||
created=date(2024, 3, 20),
|
||||
),
|
||||
content="# ACV2-200: Feature Request\n\nDetails here.\n",
|
||||
file_path=Path("acv2-200-feature-request.md"),
|
||||
),
|
||||
],
|
||||
errors=[],
|
||||
skipped=0,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=knowledge_root,
|
||||
context=ctx,
|
||||
no_commit=True,
|
||||
enrichment_agent=mock_enrichment,
|
||||
source_strategies={
|
||||
"confluence": mock_confluence,
|
||||
"jira": mock_jira,
|
||||
},
|
||||
)
|
||||
|
||||
result = pipeline.run()
|
||||
|
||||
# Both sources should contribute artifacts
|
||||
assert result.processed == 2
|
||||
assert result.context == ctx
|
||||
assert len(result.errors) == 0
|
||||
|
||||
# Index should be non-trivial
|
||||
index_content = (kp / "_index.yaml").read_text(encoding="utf-8")
|
||||
assert "artifacts" in index_content
|
||||
|
||||
def test_pipeline_partial_source_failure(self, knowledge_root: Path) -> None:
|
||||
"""Pipeline continues processing when one source fails (fail-forward)."""
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
from monorepo.knowledge.ingestion.enrichment import EnrichmentAgent, EnrichmentResult
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.base import ExtractionResult, SourceError
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from datetime import date
|
||||
|
||||
ctx = "bahn"
|
||||
kp = knowledge_root / ctx / "knowledge"
|
||||
|
||||
# sources.yaml with two sources – first will fail
|
||||
sources_yaml = {
|
||||
"sources": [
|
||||
{
|
||||
"type": "confluence",
|
||||
"name": "failing-source",
|
||||
"enabled": True,
|
||||
"params": {"base_url": "https://broken.example.com", "token": "x"},
|
||||
},
|
||||
{
|
||||
"type": "jira",
|
||||
"name": "working-source",
|
||||
"enabled": True,
|
||||
"params": {
|
||||
"base_url": "https://jira.bahn.de",
|
||||
"token": "x",
|
||||
"jql": "project = X",
|
||||
},
|
||||
},
|
||||
],
|
||||
"settings": {
|
||||
"create_tasks": False,
|
||||
"git": {"auto_commit": False},
|
||||
},
|
||||
}
|
||||
(kp / "sources.yaml").write_text(
|
||||
yaml.dump(sources_yaml), encoding="utf-8"
|
||||
)
|
||||
|
||||
# Mock enrichment
|
||||
mock_enrichment = MagicMock(spec=EnrichmentAgent)
|
||||
mock_enrichment.enrich.return_value = EnrichmentResult(
|
||||
title="Working Artifact",
|
||||
category="reference",
|
||||
tags=[],
|
||||
people=[],
|
||||
projects=[],
|
||||
action_items=[],
|
||||
confidence=0.8,
|
||||
)
|
||||
|
||||
# Confluence raises an exception
|
||||
mock_confluence = MagicMock(spec=ConfluenceSource)
|
||||
mock_confluence.extract.side_effect = ConnectionError("API unreachable")
|
||||
|
||||
# Jira returns a valid artifact
|
||||
mock_jira = MagicMock(spec=JiraSource)
|
||||
mock_jira.extract.return_value = ExtractionResult(
|
||||
artifacts=[
|
||||
Artifact(
|
||||
metadata=ArtifactMetadata(
|
||||
type="reference",
|
||||
title="Surviving Artifact",
|
||||
tags=["jira"],
|
||||
source_context=ctx,
|
||||
source={"type": "jira", "issue_key": "X-1"},
|
||||
created=date(2024, 3, 20),
|
||||
),
|
||||
content="# Surviving Artifact\n\nStill processed.\n",
|
||||
file_path=Path("x-1-surviving.md"),
|
||||
),
|
||||
],
|
||||
errors=[],
|
||||
skipped=0,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=knowledge_root,
|
||||
context=ctx,
|
||||
no_commit=True,
|
||||
enrichment_agent=mock_enrichment,
|
||||
source_strategies={
|
||||
"confluence": mock_confluence,
|
||||
"jira": mock_jira,
|
||||
},
|
||||
)
|
||||
|
||||
result = pipeline.run()
|
||||
|
||||
# Should still process the working source
|
||||
assert result.processed >= 1
|
||||
# Should log the failure
|
||||
assert len(result.errors) >= 1
|
||||
assert any("unreachable" in e.message.lower() or "API" in e.message for e in result.errors)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test: Integration Stack Factory (create_integrated_stack)
|
||||
# Requirements: 2.2, 9.3, 9.4, 10.10
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
"""Unit-Tests für die EmailSource-Quellstrategie.
|
||||
|
||||
Testet .eml-Parsing, Attachment-Erkennung und Artefakt-Generierung.
|
||||
|
||||
Requirements: 7.1, 7.2, 7.5, 7.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
from monorepo.knowledge.sources.email import (
|
||||
EmailSource,
|
||||
_extract_body,
|
||||
_format_recipients,
|
||||
_parse_email_date,
|
||||
_slugify,
|
||||
_strip_html,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def email_dir(tmp_path: Path) -> Path:
|
||||
"""Creates a temporary directory with sample .eml files."""
|
||||
email_dir = tmp_path / "emails"
|
||||
email_dir.mkdir()
|
||||
return email_dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_eml_content() -> str:
|
||||
"""A minimal valid .eml file content."""
|
||||
return (
|
||||
"From: sender@example.com\r\n"
|
||||
"To: recipient@example.com\r\n"
|
||||
"Cc: cc@example.com\r\n"
|
||||
"Subject: Test Email Subject\r\n"
|
||||
"Date: Mon, 15 Jan 2025 10:30:00 +0100\r\n"
|
||||
"Message-ID: <test123@example.com>\r\n"
|
||||
"Content-Type: text/plain; charset=utf-8\r\n"
|
||||
"\r\n"
|
||||
"This is the email body.\r\n"
|
||||
"It has multiple lines.\r\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def multipart_eml_content() -> str:
|
||||
"""A multipart .eml file with text and attachment."""
|
||||
return (
|
||||
"From: alice@example.com\r\n"
|
||||
"To: bob@example.com\r\n"
|
||||
"Subject: Meeting Notes\r\n"
|
||||
"Date: Wed, 20 Feb 2025 14:00:00 +0000\r\n"
|
||||
"Message-ID: <meeting456@example.com>\r\n"
|
||||
"MIME-Version: 1.0\r\n"
|
||||
'Content-Type: multipart/mixed; boundary="boundary123"\r\n'
|
||||
"\r\n"
|
||||
"--boundary123\r\n"
|
||||
"Content-Type: text/plain; charset=utf-8\r\n"
|
||||
"\r\n"
|
||||
"Please find the notes attached.\r\n"
|
||||
"--boundary123\r\n"
|
||||
"Content-Type: application/pdf\r\n"
|
||||
'Content-Disposition: attachment; filename="notes.pdf"\r\n'
|
||||
"\r\n"
|
||||
"FAKE PDF CONTENT\r\n"
|
||||
"--boundary123--\r\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_config(email_dir: Path) -> SourceConfig:
|
||||
"""A SourceConfig pointing to the email directory."""
|
||||
return SourceConfig(
|
||||
type="email",
|
||||
name="Test Emails",
|
||||
params={"directory": str(email_dir)},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSlugify:
|
||||
def test_simple_text(self) -> None:
|
||||
assert _slugify("Hello World") == "hello-world"
|
||||
|
||||
def test_special_chars_removed(self) -> None:
|
||||
assert _slugify("Re: Meeting (Important!)") == "re-meeting-important"
|
||||
|
||||
def test_truncates_at_80_chars(self) -> None:
|
||||
long_text = "a" * 100
|
||||
assert len(_slugify(long_text)) == 80
|
||||
|
||||
def test_empty_string(self) -> None:
|
||||
assert _slugify("") == "untitled"
|
||||
|
||||
def test_only_special_chars(self) -> None:
|
||||
assert _slugify("!!!???") == "untitled"
|
||||
|
||||
|
||||
class TestParseEmailDate:
|
||||
def test_valid_rfc2822_date(self) -> None:
|
||||
result = _parse_email_date("Mon, 15 Jan 2025 10:30:00 +0100")
|
||||
assert result is not None
|
||||
assert result.year == 2025
|
||||
assert result.month == 1
|
||||
assert result.day == 15
|
||||
|
||||
def test_none_input(self) -> None:
|
||||
assert _parse_email_date(None) is None
|
||||
|
||||
def test_empty_string(self) -> None:
|
||||
assert _parse_email_date("") is None
|
||||
|
||||
def test_invalid_date(self) -> None:
|
||||
assert _parse_email_date("not a date") is None
|
||||
|
||||
|
||||
class TestFormatRecipients:
|
||||
def test_single_address(self) -> None:
|
||||
result = _format_recipients("user@example.com")
|
||||
assert result == ["user@example.com"]
|
||||
|
||||
def test_named_address(self) -> None:
|
||||
result = _format_recipients("John Doe <john@example.com>")
|
||||
assert result == ["John Doe <john@example.com>"]
|
||||
|
||||
def test_multiple_addresses(self) -> None:
|
||||
result = _format_recipients("a@x.com, b@x.com")
|
||||
assert len(result) == 2
|
||||
|
||||
def test_none_input(self) -> None:
|
||||
assert _format_recipients(None) == []
|
||||
|
||||
def test_empty_string(self) -> None:
|
||||
assert _format_recipients("") == []
|
||||
|
||||
|
||||
class TestStripHtml:
|
||||
def test_basic_tags(self) -> None:
|
||||
result = _strip_html("<p>Hello <b>world</b></p>")
|
||||
assert "Hello" in result
|
||||
assert "world" in result
|
||||
assert "<" not in result
|
||||
|
||||
def test_br_tags(self) -> None:
|
||||
result = _strip_html("Line1<br>Line2")
|
||||
assert "Line1" in result
|
||||
assert "Line2" in result
|
||||
|
||||
def test_script_removal(self) -> None:
|
||||
result = _strip_html("<script>alert('x')</script>Content")
|
||||
assert "alert" not in result
|
||||
assert "Content" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: EmailSource.extract()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEmailSourceExtract:
|
||||
def test_extracts_simple_eml(
|
||||
self, email_dir: Path, source_config: SourceConfig, sample_eml_content: str
|
||||
) -> None:
|
||||
"""Test that a simple .eml file is correctly parsed into an artifact."""
|
||||
(email_dir / "test.eml").write_bytes(sample_eml_content.encode("utf-8"))
|
||||
|
||||
source = EmailSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
assert len(result.errors) == 0
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.title == "Test Email Subject"
|
||||
assert artifact.metadata.type == "email"
|
||||
assert artifact.metadata.source["type"] == "email"
|
||||
assert "sender@example.com" in artifact.metadata.people
|
||||
assert artifact.metadata.created is not None
|
||||
assert artifact.metadata.created.year == 2025
|
||||
assert "This is the email body." in artifact.content
|
||||
|
||||
def test_extracts_multipart_with_attachment(
|
||||
self, email_dir: Path, source_config: SourceConfig, multipart_eml_content: str
|
||||
) -> None:
|
||||
"""Test multipart email with attachment creates artifact with links."""
|
||||
(email_dir / "meeting.eml").write_bytes(multipart_eml_content.encode("utf-8"))
|
||||
|
||||
source = EmailSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.title == "Meeting Notes"
|
||||
assert "notes.pdf" in artifact.content
|
||||
assert "## Anhänge" in artifact.content
|
||||
|
||||
def test_empty_directory(self, email_dir: Path, source_config: SourceConfig) -> None:
|
||||
"""Test that an empty directory returns no artifacts."""
|
||||
source = EmailSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 0
|
||||
|
||||
def test_nonexistent_directory(self, tmp_path: Path) -> None:
|
||||
"""Test error handling for non-existent directory."""
|
||||
config = SourceConfig(
|
||||
type="email",
|
||||
name="Missing",
|
||||
params={"directory": str(tmp_path / "nonexistent")},
|
||||
)
|
||||
source = EmailSource()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 1
|
||||
assert "existiert nicht" in result.errors[0].message
|
||||
|
||||
def test_non_email_files_ignored(
|
||||
self, email_dir: Path, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Test that non-email files are ignored."""
|
||||
(email_dir / "readme.txt").write_text("Not an email")
|
||||
(email_dir / "data.csv").write_text("a,b,c")
|
||||
|
||||
source = EmailSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
|
||||
def test_multiple_eml_files(
|
||||
self, email_dir: Path, source_config: SourceConfig, sample_eml_content: str
|
||||
) -> None:
|
||||
"""Test processing multiple .eml files."""
|
||||
(email_dir / "first.eml").write_bytes(sample_eml_content.encode("utf-8"))
|
||||
(email_dir / "second.eml").write_bytes(
|
||||
sample_eml_content.replace("Test Email Subject", "Second Email").encode("utf-8")
|
||||
)
|
||||
|
||||
source = EmailSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 2
|
||||
|
||||
def test_frontmatter_contains_required_fields(
|
||||
self, email_dir: Path, source_config: SourceConfig, sample_eml_content: str
|
||||
) -> None:
|
||||
"""Test that artifact frontmatter has sender, recipients, date, source."""
|
||||
(email_dir / "test.eml").write_bytes(sample_eml_content.encode("utf-8"))
|
||||
|
||||
source = EmailSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
# Check source dict
|
||||
assert artifact.metadata.source["type"] == "email"
|
||||
assert "message_id" in artifact.metadata.source
|
||||
# Check people includes sender and recipients
|
||||
assert "sender@example.com" in artifact.metadata.people
|
||||
assert "recipient@example.com" in artifact.metadata.people
|
||||
# Check date
|
||||
assert artifact.metadata.created is not None
|
||||
|
||||
def test_supports_incremental_returns_false(self) -> None:
|
||||
"""Email source does not support incremental processing."""
|
||||
source = EmailSource()
|
||||
assert source.supports_incremental() is False
|
||||
|
||||
def test_content_hash_is_set(
|
||||
self, email_dir: Path, source_config: SourceConfig, sample_eml_content: str
|
||||
) -> None:
|
||||
"""Test that content hash is computed."""
|
||||
(email_dir / "test.eml").write_bytes(sample_eml_content.encode("utf-8"))
|
||||
|
||||
source = EmailSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.content_hash.startswith("sha256:")
|
||||
|
||||
def test_malformed_eml_produces_error(
|
||||
self, email_dir: Path, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Test that a completely invalid file doesn't crash but may produce minimal artifact."""
|
||||
# An empty file is still parseable by Python's email module (produces empty Message)
|
||||
(email_dir / "broken.eml").write_bytes(b"")
|
||||
|
||||
source = EmailSource()
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
# Should still produce an artifact (with default values) or an error
|
||||
assert len(result.artifacts) + len(result.errors) >= 0 # No crash
|
||||
@@ -0,0 +1,542 @@
|
||||
"""Tests for EnrichmentAgent – KI-gestützte Anreicherung.
|
||||
|
||||
Requirements: 9.1, 9.2, 9.3, 9.4, 9.5, 9.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.ingestion.enrichment import (
|
||||
CONFIDENCE_THRESHOLD,
|
||||
ActionItem,
|
||||
EnrichmentAgent,
|
||||
EnrichmentResult,
|
||||
_compute_action_item_hash,
|
||||
_detect_category,
|
||||
_extract_action_items,
|
||||
_extract_people,
|
||||
_extract_projects,
|
||||
_extract_title,
|
||||
_slugify,
|
||||
generate_action_items_section,
|
||||
generate_wiki_links,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _slugify tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSlugify:
|
||||
"""Tests for _slugify helper."""
|
||||
|
||||
def test_simple_name(self) -> None:
|
||||
assert _slugify("Max Müller") == "max-mueller"
|
||||
|
||||
def test_umlaut_handling(self) -> None:
|
||||
assert _slugify("André Knie") == "andre-knie"
|
||||
|
||||
def test_sharp_s(self) -> None:
|
||||
assert _slugify("Hans Straße") == "hans-strasse"
|
||||
|
||||
def test_multiple_spaces(self) -> None:
|
||||
assert _slugify("Max Müller") == "max-mueller"
|
||||
|
||||
def test_leading_trailing_spaces(self) -> None:
|
||||
assert _slugify(" Max Müller ") == "max-mueller"
|
||||
|
||||
def test_single_word(self) -> None:
|
||||
assert _slugify("Monorepo") == "monorepo"
|
||||
|
||||
def test_special_characters(self) -> None:
|
||||
assert _slugify("Projekt (v2.0)") == "projekt-v2-0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _compute_action_item_hash tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestComputeActionItemHash:
|
||||
"""Tests for _compute_action_item_hash."""
|
||||
|
||||
def test_deterministic(self) -> None:
|
||||
h1 = _compute_action_item_hash("Do the thing")
|
||||
h2 = _compute_action_item_hash("Do the thing")
|
||||
assert h1 == h2
|
||||
|
||||
def test_case_insensitive(self) -> None:
|
||||
h1 = _compute_action_item_hash("Do the thing")
|
||||
h2 = _compute_action_item_hash("do the thing")
|
||||
assert h1 == h2
|
||||
|
||||
def test_trims_whitespace(self) -> None:
|
||||
h1 = _compute_action_item_hash("Do the thing")
|
||||
h2 = _compute_action_item_hash(" Do the thing ")
|
||||
assert h1 == h2
|
||||
|
||||
def test_different_descriptions_differ(self) -> None:
|
||||
h1 = _compute_action_item_hash("Do thing A")
|
||||
h2 = _compute_action_item_hash("Do thing B")
|
||||
assert h1 != h2
|
||||
|
||||
def test_returns_16_chars(self) -> None:
|
||||
h = _compute_action_item_hash("some task")
|
||||
assert len(h) == 16
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _detect_category tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDetectCategory:
|
||||
"""Tests for _detect_category."""
|
||||
|
||||
def test_meeting_keywords(self) -> None:
|
||||
content = "Meeting Protokoll vom 15.01.2025\nTeilnehmer: Max, Anna"
|
||||
category, confidence = _detect_category(content)
|
||||
assert category == "meeting"
|
||||
assert confidence >= 0.5
|
||||
|
||||
def test_decision_keywords(self) -> None:
|
||||
content = "Entscheidung: Wir haben beschlossen, die API umzubauen."
|
||||
category, _ = _detect_category(content)
|
||||
assert category == "decision"
|
||||
|
||||
def test_project_keywords(self) -> None:
|
||||
content = "Projekt Roadmap Q1 2025\nMeilenstein 1: Release Sprint 3"
|
||||
category, _ = _detect_category(content)
|
||||
assert category == "project"
|
||||
|
||||
def test_reference_keywords(self) -> None:
|
||||
content = "Dokumentation: How-To Guide für den Deployment-Prozess"
|
||||
category, _ = _detect_category(content)
|
||||
assert category == "reference"
|
||||
|
||||
def test_link_detection(self) -> None:
|
||||
content = "https://docs.python.org/3/library/re.html"
|
||||
category, confidence = _detect_category(content)
|
||||
assert category == "link"
|
||||
assert confidence >= 0.8
|
||||
|
||||
def test_unknown_content_defaults_inbox(self) -> None:
|
||||
content = "Einfach nur ein Gedanke."
|
||||
category, confidence = _detect_category(content)
|
||||
assert category == "inbox"
|
||||
assert confidence < CONFIDENCE_THRESHOLD
|
||||
|
||||
def test_source_type_hint(self) -> None:
|
||||
content = "Some content about things"
|
||||
category, _ = _detect_category(content, source_type="jira")
|
||||
assert category == "project"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _extract_people tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractPeople:
|
||||
"""Tests for _extract_people."""
|
||||
|
||||
def test_at_mention(self) -> None:
|
||||
content = "Assigned to @Max Müller for review."
|
||||
people = _extract_people(content)
|
||||
names = [n for n, _ in people]
|
||||
assert any("Max" in n for n in names)
|
||||
|
||||
def test_name_pattern(self) -> None:
|
||||
content = "Besprochen mit André Knie und Hans Schmidt."
|
||||
people = _extract_people(content)
|
||||
names = [n for n, _ in people]
|
||||
assert any("André Knie" in n for n in names)
|
||||
assert any("Hans Schmidt" in n for n in names)
|
||||
|
||||
def test_excludes_stop_names(self) -> None:
|
||||
content = "Meeting Notes und Action Items besprochen."
|
||||
people = _extract_people(content)
|
||||
names = [n for n, _ in people]
|
||||
assert "Action Items" not in names
|
||||
assert "Meeting Notes" not in names
|
||||
|
||||
def test_empty_content(self) -> None:
|
||||
people = _extract_people("")
|
||||
assert people == []
|
||||
|
||||
def test_at_mention_high_confidence(self) -> None:
|
||||
content = "Feedback von @André eingereicht."
|
||||
people = _extract_people(content)
|
||||
assert any(conf >= 0.9 for _, conf in people)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _extract_projects tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractProjects:
|
||||
"""Tests for _extract_projects."""
|
||||
|
||||
def test_explicit_project_reference(self) -> None:
|
||||
content = "Betrifft Projekt Orchestrator-Monorepo."
|
||||
projects = _extract_projects(content)
|
||||
names = [n for n, _ in projects]
|
||||
assert any("Orchestrator-Monorepo" in n for n in names)
|
||||
|
||||
def test_english_project_reference(self) -> None:
|
||||
content = "This relates to Project Knowledge-Pipeline."
|
||||
projects = _extract_projects(content)
|
||||
names = [n for n, _ in projects]
|
||||
assert any("Knowledge-Pipeline" in n for n in names)
|
||||
|
||||
def test_existing_wiki_link(self) -> None:
|
||||
content = "See [[projects/my-project]] for details."
|
||||
projects = _extract_projects(content)
|
||||
names = [n for n, _ in projects]
|
||||
assert "my-project" in names
|
||||
|
||||
def test_empty_content(self) -> None:
|
||||
projects = _extract_projects("")
|
||||
assert projects == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _extract_action_items tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractActionItems:
|
||||
"""Tests for _extract_action_items."""
|
||||
|
||||
def test_checkbox_pattern(self) -> None:
|
||||
content = "Tasks:\n- [ ] Finish the report\n- [ ] Send email"
|
||||
items = _extract_action_items(content)
|
||||
assert len(items) == 2
|
||||
assert items[0].description == "Finish the report"
|
||||
assert items[1].description == "Send email"
|
||||
|
||||
def test_todo_pattern(self) -> None:
|
||||
content = "TODO: Review PR #42\nSome other text"
|
||||
items = _extract_action_items(content)
|
||||
assert len(items) == 1
|
||||
assert "Review PR #42" in items[0].description
|
||||
|
||||
def test_action_pattern(self) -> None:
|
||||
content = "ACTION: Schedule follow-up meeting"
|
||||
items = _extract_action_items(content)
|
||||
assert len(items) == 1
|
||||
assert "Schedule follow-up meeting" in items[0].description
|
||||
|
||||
def test_arrow_pattern(self) -> None:
|
||||
content = "→ Create Jira ticket for this"
|
||||
items = _extract_action_items(content)
|
||||
assert len(items) == 1
|
||||
assert "Create Jira ticket" in items[0].description
|
||||
|
||||
def test_deduplication(self) -> None:
|
||||
content = "- [ ] Do the thing\n- [ ] Do the thing"
|
||||
items = _extract_action_items(content)
|
||||
assert len(items) == 1
|
||||
|
||||
def test_empty_content(self) -> None:
|
||||
items = _extract_action_items("")
|
||||
assert items == []
|
||||
|
||||
def test_assignee_detection(self) -> None:
|
||||
content = "- [ ] Review document @André"
|
||||
items = _extract_action_items(content)
|
||||
assert len(items) == 1
|
||||
assert items[0].assignee == "André"
|
||||
|
||||
def test_deadline_detection(self) -> None:
|
||||
content = "- [ ] Submit report bis 2025-03-15"
|
||||
items = _extract_action_items(content)
|
||||
assert len(items) == 1
|
||||
assert items[0].deadline == "2025-03-15"
|
||||
|
||||
def test_hash_auto_computed(self) -> None:
|
||||
content = "TODO: Something important"
|
||||
items = _extract_action_items(content)
|
||||
assert items[0].hash != ""
|
||||
assert len(items[0].hash) == 16
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _extract_title tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractTitle:
|
||||
"""Tests for _extract_title."""
|
||||
|
||||
def test_markdown_heading(self) -> None:
|
||||
content = "# My Document Title\n\nSome content here."
|
||||
assert _extract_title(content) == "My Document Title"
|
||||
|
||||
def test_h2_heading(self) -> None:
|
||||
content = "## Second Level Heading\n\nContent."
|
||||
assert _extract_title(content) == "Second Level Heading"
|
||||
|
||||
def test_first_line_fallback(self) -> None:
|
||||
content = "Just some text without a heading\nMore text."
|
||||
title = _extract_title(content)
|
||||
assert "Just some text" in title
|
||||
|
||||
def test_empty_content(self) -> None:
|
||||
assert _extract_title("") == "Untitled"
|
||||
|
||||
def test_whitespace_only(self) -> None:
|
||||
assert _extract_title(" \n\n ") == "Untitled"
|
||||
|
||||
def test_long_first_line_truncated(self) -> None:
|
||||
content = "A" * 200
|
||||
title = _extract_title(content)
|
||||
assert len(title) <= 80
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# generate_wiki_links tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGenerateWikiLinks:
|
||||
"""Tests for generate_wiki_links."""
|
||||
|
||||
def test_people_links(self) -> None:
|
||||
links = generate_wiki_links(["Max Müller", "André Knie"], [])
|
||||
assert links["people"] == [
|
||||
"[[people/max-mueller]]",
|
||||
"[[people/andre-knie]]",
|
||||
]
|
||||
|
||||
def test_project_links(self) -> None:
|
||||
links = generate_wiki_links([], ["Knowledge Pipeline"])
|
||||
assert links["projects"] == ["[[projects/knowledge-pipeline]]"]
|
||||
|
||||
def test_empty_lists(self) -> None:
|
||||
links = generate_wiki_links([], [])
|
||||
assert links["people"] == []
|
||||
assert links["projects"] == []
|
||||
|
||||
def test_combined(self) -> None:
|
||||
links = generate_wiki_links(["Hans Schmidt"], ["Orchestrator"])
|
||||
assert "[[people/hans-schmidt]]" in links["people"]
|
||||
assert "[[projects/orchestrator]]" in links["projects"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# generate_action_items_section tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGenerateActionItemsSection:
|
||||
"""Tests for generate_action_items_section."""
|
||||
|
||||
def test_empty_list(self) -> None:
|
||||
assert generate_action_items_section([]) == ""
|
||||
|
||||
def test_single_item(self) -> None:
|
||||
items = [ActionItem(description="Do the thing")]
|
||||
section = generate_action_items_section(items)
|
||||
assert "## Action Items" in section
|
||||
assert "- [ ] Do the thing" in section
|
||||
|
||||
def test_item_with_assignee(self) -> None:
|
||||
items = [ActionItem(description="Review code", assignee="Max")]
|
||||
section = generate_action_items_section(items)
|
||||
assert "@Max" in section
|
||||
|
||||
def test_item_with_deadline(self) -> None:
|
||||
items = [ActionItem(description="Submit report", deadline="2025-03-15")]
|
||||
section = generate_action_items_section(items)
|
||||
assert "bis 2025-03-15" in section
|
||||
|
||||
def test_multiple_items(self) -> None:
|
||||
items = [
|
||||
ActionItem(description="Task A"),
|
||||
ActionItem(description="Task B"),
|
||||
ActionItem(description="Task C"),
|
||||
]
|
||||
section = generate_action_items_section(items)
|
||||
assert section.count("- [ ]") == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EnrichmentAgent tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnrichmentAgent:
|
||||
"""Tests for EnrichmentAgent class."""
|
||||
|
||||
@pytest.fixture
|
||||
def agent(self) -> EnrichmentAgent:
|
||||
return EnrichmentAgent(provider="kiro")
|
||||
|
||||
def test_default_provider_is_kiro(self) -> None:
|
||||
agent = EnrichmentAgent()
|
||||
assert agent.provider == "kiro"
|
||||
|
||||
def test_provider_available_for_kiro(self) -> None:
|
||||
agent = EnrichmentAgent(provider="kiro")
|
||||
assert agent._provider_available is True
|
||||
|
||||
def test_provider_unavailable_for_litellm(self) -> None:
|
||||
agent = EnrichmentAgent(provider="litellm")
|
||||
assert agent._provider_available is False
|
||||
|
||||
def test_enrich_empty_content(self, agent: EnrichmentAgent) -> None:
|
||||
result = agent.enrich("")
|
||||
assert result.title == "Untitled"
|
||||
assert result.category == "inbox"
|
||||
assert result.confidence == 0.0
|
||||
|
||||
def test_enrich_meeting_content(self, agent: EnrichmentAgent) -> None:
|
||||
content = """# Meeting Protokoll 2025-01-15
|
||||
|
||||
Teilnehmer: André Knie, Max Müller
|
||||
|
||||
## Agenda
|
||||
1. Sprint Review
|
||||
2. Nächste Schritte
|
||||
|
||||
## Action Items
|
||||
- [ ] André erstellt Jira-Ticket
|
||||
- [ ] Max reviewed den PR bis 2025-01-20
|
||||
"""
|
||||
result = agent.enrich(content)
|
||||
assert result.title == "Meeting Protokoll 2025-01-15"
|
||||
assert result.category == "meeting"
|
||||
assert result.confidence > 0.5
|
||||
assert len(result.action_items) >= 2
|
||||
assert result.enrichment_pending is False
|
||||
|
||||
def test_enrich_detects_people(self, agent: EnrichmentAgent) -> None:
|
||||
content = "Besprochen mit André Knie und Hans Schmidt im Meeting."
|
||||
result = agent.enrich(content)
|
||||
assert len(result.people) >= 1
|
||||
|
||||
def test_enrich_detects_projects(self, agent: EnrichmentAgent) -> None:
|
||||
content = "Betrifft Projekt Orchestrator-Monorepo und die Pipeline."
|
||||
result = agent.enrich(content)
|
||||
assert len(result.projects) >= 1
|
||||
|
||||
def test_enrich_confidence_threshold(self, agent: EnrichmentAgent) -> None:
|
||||
"""Entities below confidence threshold are excluded."""
|
||||
content = "Some text mentioning @Max in a clear mention."
|
||||
result = agent.enrich(content)
|
||||
# All returned people must have passed confidence threshold
|
||||
# (tested indirectly: if people are returned, they passed)
|
||||
for person in result.people:
|
||||
assert isinstance(person, str)
|
||||
assert len(person) >= 2
|
||||
|
||||
def test_enrich_pending_when_provider_unavailable(self) -> None:
|
||||
agent = EnrichmentAgent(provider="litellm")
|
||||
result = agent.enrich("Some content to analyze")
|
||||
assert result.enrichment_pending is True
|
||||
assert result.category == "inbox"
|
||||
|
||||
def test_enrich_returns_tags(self, agent: EnrichmentAgent) -> None:
|
||||
content = "# Python API Documentation\n\nUsing Docker and Kubernetes for deployment."
|
||||
result = agent.enrich(content)
|
||||
assert "python" in result.tags or "docker" in result.tags or "kubernetes" in result.tags
|
||||
|
||||
def test_enrich_link_category(self, agent: EnrichmentAgent) -> None:
|
||||
content = "https://docs.python.org/3/library/re.html"
|
||||
result = agent.enrich(content)
|
||||
assert result.category == "link"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# classify_relevance tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestClassifyRelevance:
|
||||
"""Tests for classify_relevance."""
|
||||
|
||||
@pytest.fixture
|
||||
def agent(self) -> EnrichmentAgent:
|
||||
return EnrichmentAgent(provider="kiro")
|
||||
|
||||
def test_empty_content_zero(self, agent: EnrichmentAgent) -> None:
|
||||
assert agent.classify_relevance("") == 0.0
|
||||
|
||||
def test_whitespace_only_zero(self, agent: EnrichmentAgent) -> None:
|
||||
assert agent.classify_relevance(" ") == 0.0
|
||||
|
||||
def test_short_content_low_score(self, agent: EnrichmentAgent) -> None:
|
||||
score = agent.classify_relevance("Hi")
|
||||
assert score <= 0.3
|
||||
|
||||
def test_structured_content_higher_score(self, agent: EnrichmentAgent) -> None:
|
||||
content = """# Important Document
|
||||
|
||||
This is a longer document with:
|
||||
- List items
|
||||
- More structure
|
||||
|
||||
## Section Two
|
||||
|
||||
Some [linked content](https://example.com) here.
|
||||
|
||||
TODO: Do something important
|
||||
"""
|
||||
score = agent.classify_relevance(content)
|
||||
assert score >= 0.6
|
||||
|
||||
def test_score_between_zero_and_one(self, agent: EnrichmentAgent) -> None:
|
||||
content = "Medium length content with some words and information."
|
||||
score = agent.classify_relevance(content)
|
||||
assert 0.0 <= score <= 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ActionItem dataclass tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestActionItem:
|
||||
"""Tests for ActionItem dataclass."""
|
||||
|
||||
def test_auto_hash_on_creation(self) -> None:
|
||||
item = ActionItem(description="Do something")
|
||||
assert item.hash != ""
|
||||
assert len(item.hash) == 16
|
||||
|
||||
def test_preserves_explicit_hash(self) -> None:
|
||||
item = ActionItem(description="Do something", hash="custom_hash_val!")
|
||||
assert item.hash == "custom_hash_val!"
|
||||
|
||||
def test_optional_fields_default_none(self) -> None:
|
||||
item = ActionItem(description="Task")
|
||||
assert item.assignee is None
|
||||
assert item.deadline is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EnrichmentResult dataclass tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnrichmentResult:
|
||||
"""Tests for EnrichmentResult dataclass."""
|
||||
|
||||
def test_defaults(self) -> None:
|
||||
result = EnrichmentResult(title="Test", category="inbox")
|
||||
assert result.tags == []
|
||||
assert result.people == []
|
||||
assert result.projects == []
|
||||
assert result.action_items == []
|
||||
assert result.confidence == 0.0
|
||||
assert result.enrichment_pending is False
|
||||
|
||||
def test_enrichment_pending_flag(self) -> None:
|
||||
result = EnrichmentResult(
|
||||
title="Test", category="inbox", enrichment_pending=True
|
||||
)
|
||||
assert result.enrichment_pending is True
|
||||
@@ -0,0 +1,418 @@
|
||||
"""Property-Based Tests für Enrichment Agent.
|
||||
|
||||
**Validates: Requirements 9.2, 9.4, 9.5**
|
||||
|
||||
Property 15: Wiki-Link Format for Entities
|
||||
*For any* person name detected by the Enrichment Agent, the generated wiki-link
|
||||
SHALL match the format `[[people/{slugified-name}]]`. For any project name, it
|
||||
SHALL match `[[projects/{slugified-name}]]`. Slugification SHALL produce
|
||||
lowercase, hyphen-separated strings.
|
||||
|
||||
Property 16: Confidence Threshold Filtering
|
||||
*For any* set of detected entities with varying confidence scores, only entities
|
||||
with confidence >= 0.7 SHALL appear in the final enrichment output. Entities
|
||||
below 0.7 SHALL be excluded.
|
||||
|
||||
Property 17: Action Items Section Generation
|
||||
*For any* non-empty list of `ActionItem` objects, the generated markdown SHALL
|
||||
contain a `## Action Items` section header followed by a markdown list item for
|
||||
each action item's description.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from hypothesis import given, settings, assume, HealthCheck
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.ingestion.enrichment import (
|
||||
CONFIDENCE_THRESHOLD,
|
||||
ActionItem,
|
||||
EnrichmentAgent,
|
||||
_slugify,
|
||||
generate_action_items_section,
|
||||
generate_wiki_links,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Person name strategy: "Vorname Nachname" with Latin characters
|
||||
_name_part_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Lu", "Ll"),
|
||||
max_codepoint=255, # Latin-1 range for realistic names
|
||||
),
|
||||
min_size=3,
|
||||
max_size=12,
|
||||
).map(lambda s: s[0].upper() + s[1:].lower() if len(s) >= 2 else s.capitalize())
|
||||
|
||||
|
||||
@st.composite
|
||||
def person_name_st(draw: st.DrawFn) -> str:
|
||||
"""Generate a realistic 'Vorname Nachname' string."""
|
||||
first = draw(_name_part_st)
|
||||
last = draw(_name_part_st)
|
||||
assume(len(first) >= 3 and len(last) >= 3)
|
||||
assume(first[0].isupper() and last[0].isupper())
|
||||
name = f"{first} {last}"
|
||||
# Ensure slug is at least 2 chars
|
||||
slug = _slugify(name)
|
||||
assume(len(slug) >= 2)
|
||||
return name
|
||||
|
||||
|
||||
@st.composite
|
||||
def project_name_st(draw: st.DrawFn) -> str:
|
||||
"""Generate a project name (alphanumeric with hyphens/spaces).
|
||||
|
||||
Ensures slugified result has at least 2 chars (realistic project names).
|
||||
"""
|
||||
parts = draw(st.lists(
|
||||
st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Lu"),
|
||||
whitelist_characters="-",
|
||||
),
|
||||
min_size=3,
|
||||
max_size=12,
|
||||
).filter(lambda s: any(c.isalpha() for c in s)),
|
||||
min_size=1,
|
||||
max_size=3,
|
||||
))
|
||||
name = " ".join(parts)
|
||||
assume(len(name) >= 3)
|
||||
# Ensure slug would be at least 2 chars
|
||||
slug = _slugify(name)
|
||||
assume(len(slug) >= 2)
|
||||
return name
|
||||
|
||||
|
||||
# Action item description strategy
|
||||
_action_description_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "P", "Z"),
|
||||
blacklist_characters="\x00\r\n",
|
||||
),
|
||||
min_size=5,
|
||||
max_size=80,
|
||||
).filter(lambda s: s.strip() != "" and len(s.strip()) >= 5)
|
||||
|
||||
|
||||
# Confidence scores
|
||||
_confidence_above_st = st.floats(min_value=0.7, max_value=1.0, allow_nan=False)
|
||||
_confidence_below_st = st.floats(min_value=0.0, max_value=0.69, allow_nan=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 15: Wiki-Link Format for Entities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestWikiLinkFormatForEntities:
|
||||
"""**Validates: Requirements 9.2**
|
||||
|
||||
Property 15: For any person name, the generated wiki-link SHALL match
|
||||
`[[people/{slugified-name}]]`. For any project name, it SHALL match
|
||||
`[[projects/{slugified-name}]]`. Slugification SHALL produce lowercase,
|
||||
hyphen-separated strings.
|
||||
"""
|
||||
|
||||
@given(names=st.lists(person_name_st(), min_size=1, max_size=5))
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_people_wiki_links_have_correct_format(self, names: list[str]) -> None:
|
||||
"""All person wiki-links match [[people/{slug}]] format."""
|
||||
result = generate_wiki_links(people=names, projects=[])
|
||||
|
||||
assert len(result["people"]) == len(names)
|
||||
|
||||
for link in result["people"]:
|
||||
# Must match [[people/...]] pattern: slug is lowercase, hyphen-separated
|
||||
assert re.match(r"^\[\[people/[a-z0-9]([a-z0-9-]*[a-z0-9])?\]\]$", link), (
|
||||
f"Wiki-link {link!r} does not match expected format [[people/slug]]"
|
||||
)
|
||||
|
||||
@given(names=st.lists(project_name_st(), min_size=1, max_size=5))
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_project_wiki_links_have_correct_format(self, names: list[str]) -> None:
|
||||
"""All project wiki-links match [[projects/{slug}]] format."""
|
||||
result = generate_wiki_links(people=[], projects=names)
|
||||
|
||||
assert len(result["projects"]) == len(names)
|
||||
|
||||
for link in result["projects"]:
|
||||
# Must match [[projects/...]] pattern: slug is lowercase, hyphen-separated
|
||||
assert re.match(r"^\[\[projects/[a-z0-9]([a-z0-9-]*[a-z0-9])?\]\]$", link), (
|
||||
f"Wiki-link {link!r} does not match expected format [[projects/slug]]"
|
||||
)
|
||||
|
||||
@given(name=person_name_st())
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_slugification_is_lowercase(self, name: str) -> None:
|
||||
"""Slugified names are always lowercase."""
|
||||
slug = _slugify(name)
|
||||
assert slug == slug.lower(), (
|
||||
f"Slug {slug!r} for name {name!r} is not fully lowercase"
|
||||
)
|
||||
|
||||
@given(name=person_name_st())
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_slugification_is_hyphen_separated(self, name: str) -> None:
|
||||
"""Slugified names use hyphens as separators (no spaces, underscores, etc.)."""
|
||||
slug = _slugify(name)
|
||||
# Must contain only lowercase alphanumeric and hyphens
|
||||
assert re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", slug), (
|
||||
f"Slug {slug!r} for name {name!r} contains invalid characters "
|
||||
f"(expected only lowercase alphanumeric and hyphens)"
|
||||
)
|
||||
|
||||
@given(
|
||||
people=st.lists(person_name_st(), min_size=1, max_size=3),
|
||||
projects=st.lists(project_name_st(), min_size=1, max_size=3),
|
||||
)
|
||||
@settings(max_examples=30, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_combined_links_have_separate_namespaces(
|
||||
self, people: list[str], projects: list[str]
|
||||
) -> None:
|
||||
"""People links use 'people/' prefix and project links use 'projects/' prefix."""
|
||||
result = generate_wiki_links(people=people, projects=projects)
|
||||
|
||||
for link in result["people"]:
|
||||
assert link.startswith("[[people/"), f"Person link {link!r} missing 'people/' prefix"
|
||||
|
||||
for link in result["projects"]:
|
||||
assert link.startswith("[[projects/"), f"Project link {link!r} missing 'projects/' prefix"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 16: Confidence Threshold Filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConfidenceThresholdFiltering:
|
||||
"""**Validates: Requirements 9.5**
|
||||
|
||||
Property 16: For any set of detected entities with varying confidence
|
||||
scores, only entities with confidence >= 0.7 SHALL appear in the final
|
||||
enrichment output. Entities below 0.7 SHALL be excluded.
|
||||
"""
|
||||
|
||||
@given(
|
||||
high_conf_names=st.lists(
|
||||
st.tuples(person_name_st(), _confidence_above_st),
|
||||
min_size=0,
|
||||
max_size=3,
|
||||
),
|
||||
low_conf_names=st.lists(
|
||||
st.tuples(person_name_st(), _confidence_below_st),
|
||||
min_size=0,
|
||||
max_size=3,
|
||||
),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_only_high_confidence_entities_pass_threshold(
|
||||
self,
|
||||
high_conf_names: list[tuple[str, float]],
|
||||
low_conf_names: list[tuple[str, float]],
|
||||
) -> None:
|
||||
"""Entities with confidence >= 0.7 are included, those below are excluded."""
|
||||
assume(len(high_conf_names) + len(low_conf_names) > 0)
|
||||
# Ensure no name overlap between high and low sets (same name at different
|
||||
# confidence levels is not a meaningful test case for threshold filtering)
|
||||
high_names = {name for name, _ in high_conf_names}
|
||||
low_names = {name for name, _ in low_conf_names}
|
||||
assume(high_names.isdisjoint(low_names))
|
||||
|
||||
# Simulate the threshold filtering logic used in EnrichmentAgent.enrich()
|
||||
all_entities = high_conf_names + low_conf_names
|
||||
filtered = [name for name, conf in all_entities if conf >= CONFIDENCE_THRESHOLD]
|
||||
|
||||
# All high-confidence names should be in filtered
|
||||
for name, conf in high_conf_names:
|
||||
assert name in filtered, (
|
||||
f"High-confidence entity {name!r} (conf={conf}) was excluded"
|
||||
)
|
||||
|
||||
# No low-confidence names should be in filtered
|
||||
for name, conf in low_conf_names:
|
||||
assert name not in filtered, (
|
||||
f"Low-confidence entity {name!r} (conf={conf}) was not excluded"
|
||||
)
|
||||
|
||||
def test_threshold_constant_is_0_7(self) -> None:
|
||||
"""The confidence threshold constant is exactly 0.7."""
|
||||
assert CONFIDENCE_THRESHOLD == 0.7
|
||||
|
||||
@given(
|
||||
content=st.just(
|
||||
"Meeting mit André Knie und Hans Schmidt.\n"
|
||||
"Betrifft Projekt Knowledge-Pipeline.\n"
|
||||
"@Max assigned to review."
|
||||
)
|
||||
)
|
||||
@settings(max_examples=5, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_enrichment_agent_applies_threshold(self, content: str) -> None:
|
||||
"""The EnrichmentAgent.enrich() method applies the confidence threshold."""
|
||||
agent = EnrichmentAgent(provider="kiro")
|
||||
result = agent.enrich(content)
|
||||
|
||||
# All returned people should have had confidence >= 0.7 internally
|
||||
# We can't directly check confidence on the output (it's filtered already),
|
||||
# but we can verify no obviously-invalid entities slip through
|
||||
for person in result.people:
|
||||
assert isinstance(person, str)
|
||||
assert len(person) >= 2, (
|
||||
f"Person name {person!r} is suspiciously short (likely low-confidence noise)"
|
||||
)
|
||||
|
||||
@given(
|
||||
high_conf_projects=st.lists(
|
||||
st.tuples(project_name_st(), _confidence_above_st),
|
||||
min_size=0,
|
||||
max_size=3,
|
||||
),
|
||||
low_conf_projects=st.lists(
|
||||
st.tuples(project_name_st(), _confidence_below_st),
|
||||
min_size=0,
|
||||
max_size=3,
|
||||
),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_project_entities_also_filtered_by_threshold(
|
||||
self,
|
||||
high_conf_projects: list[tuple[str, float]],
|
||||
low_conf_projects: list[tuple[str, float]],
|
||||
) -> None:
|
||||
"""Project entities are also subject to the confidence threshold."""
|
||||
assume(len(high_conf_projects) + len(low_conf_projects) > 0)
|
||||
# Ensure no name overlap between high and low sets
|
||||
high_names = {name for name, _ in high_conf_projects}
|
||||
low_names = {name for name, _ in low_conf_projects}
|
||||
assume(high_names.isdisjoint(low_names))
|
||||
|
||||
all_entities = high_conf_projects + low_conf_projects
|
||||
filtered = [name for name, conf in all_entities if conf >= CONFIDENCE_THRESHOLD]
|
||||
|
||||
for name, conf in high_conf_projects:
|
||||
assert name in filtered
|
||||
|
||||
for name, conf in low_conf_projects:
|
||||
assert name not in filtered
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 17: Action Items Section Generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestActionItemsSectionGeneration:
|
||||
"""**Validates: Requirements 9.4**
|
||||
|
||||
Property 17: For any non-empty list of ActionItem objects, the generated
|
||||
markdown SHALL contain a `## Action Items` section header followed by a
|
||||
markdown list item for each action item's description.
|
||||
"""
|
||||
|
||||
@given(
|
||||
descriptions=st.lists(
|
||||
_action_description_st,
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_section_header_present(self, descriptions: list[str]) -> None:
|
||||
"""Generated markdown contains '## Action Items' header."""
|
||||
items = [ActionItem(description=desc) for desc in descriptions]
|
||||
section = generate_action_items_section(items)
|
||||
|
||||
assert "## Action Items" in section, (
|
||||
f"Missing '## Action Items' header in generated section:\n{section}"
|
||||
)
|
||||
|
||||
@given(
|
||||
descriptions=st.lists(
|
||||
_action_description_st,
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_each_action_item_has_list_entry(self, descriptions: list[str]) -> None:
|
||||
"""Each action item appears as a markdown list item in the section."""
|
||||
items = [ActionItem(description=desc) for desc in descriptions]
|
||||
section = generate_action_items_section(items)
|
||||
|
||||
for desc in descriptions:
|
||||
assert desc in section, (
|
||||
f"Action item description {desc!r} not found in generated section:\n{section}"
|
||||
)
|
||||
|
||||
@given(
|
||||
descriptions=st.lists(
|
||||
_action_description_st,
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_list_items_use_markdown_format(self, descriptions: list[str]) -> None:
|
||||
"""Each action item is formatted as a markdown checkbox list item '- [ ] ...'."""
|
||||
items = [ActionItem(description=desc) for desc in descriptions]
|
||||
section = generate_action_items_section(items)
|
||||
|
||||
# Count markdown list items
|
||||
list_item_count = section.count("- [ ]")
|
||||
assert list_item_count == len(descriptions), (
|
||||
f"Expected {len(descriptions)} list items but found {list_item_count} "
|
||||
f"in:\n{section}"
|
||||
)
|
||||
|
||||
@given(
|
||||
descriptions=st.lists(
|
||||
_action_description_st,
|
||||
min_size=1,
|
||||
max_size=5,
|
||||
unique=True,
|
||||
),
|
||||
assignees=st.lists(
|
||||
st.one_of(
|
||||
st.none(),
|
||||
st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L",)),
|
||||
min_size=3,
|
||||
max_size=10,
|
||||
),
|
||||
),
|
||||
min_size=1,
|
||||
max_size=5,
|
||||
),
|
||||
)
|
||||
@settings(max_examples=30, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_assignees_included_when_present(
|
||||
self, descriptions: list[str], assignees: list[str | None]
|
||||
) -> None:
|
||||
"""When an action item has an assignee, it appears with @ prefix."""
|
||||
# Align lists
|
||||
n = min(len(descriptions), len(assignees))
|
||||
items = [
|
||||
ActionItem(description=descriptions[i], assignee=assignees[i])
|
||||
for i in range(n)
|
||||
]
|
||||
section = generate_action_items_section(items)
|
||||
|
||||
for item in items:
|
||||
if item.assignee:
|
||||
assert f"@{item.assignee}" in section, (
|
||||
f"Assignee @{item.assignee} not found in section:\n{section}"
|
||||
)
|
||||
|
||||
def test_empty_list_returns_empty_string(self) -> None:
|
||||
"""An empty action items list produces an empty string (no section)."""
|
||||
assert generate_action_items_section([]) == ""
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Property-Based Test: Federation Sync Exclusion.
|
||||
|
||||
**Property 22: Federation Sync Exclusion**
|
||||
|
||||
For any set of artifacts where some have `shareable: false` in their frontmatter,
|
||||
the federation sync file list SHALL exclude ALL artifacts with `shareable: false`.
|
||||
Only artifacts with `shareable: true` (or shareable not explicitly set to false)
|
||||
SHALL be included.
|
||||
|
||||
**Validates: Requirements 14.4**
|
||||
|
||||
Uses Hypothesis to generate random combinations of artifacts with varying
|
||||
shareable values and verifies the filter always excludes shareable: false
|
||||
and includes shareable: true (assuming no other exclusion criteria like
|
||||
cross-context links or ctx-guard).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import hypothesis.strategies as st
|
||||
from hypothesis import given, settings
|
||||
|
||||
from monorepo.knowledge.integrations.federation import FederationSyncFilter
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Generate valid artifact titles (non-empty, safe for filenames)
|
||||
_artifact_title = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs"), whitelist_characters="-_"),
|
||||
min_size=1,
|
||||
max_size=40,
|
||||
).map(str.strip).filter(lambda s: len(s) > 0)
|
||||
|
||||
# Generate a safe filename slug (lowercase, hyphen-separated, unique-ish)
|
||||
_filename_slug = st.from_regex(r"[a-z][a-z0-9\-]{2,20}", fullmatch=True)
|
||||
|
||||
# Generate valid tags
|
||||
_tag = st.from_regex(r"[a-z][a-z0-9\-]{1,12}", fullmatch=True)
|
||||
_tags_list = st.lists(_tag, min_size=0, max_size=4)
|
||||
|
||||
# Categories that don't trigger any cross-context logic
|
||||
_category = st.sampled_from(["meetings", "decisions", "inbox", "projects", "references", "links"])
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_spec(draw: st.DrawFn) -> dict:
|
||||
"""Generate a single artifact specification with shareable true or false."""
|
||||
return {
|
||||
"filename": draw(_filename_slug) + ".md",
|
||||
"title": draw(_artifact_title),
|
||||
"shareable": draw(st.booleans()),
|
||||
"tags": draw(_tags_list),
|
||||
"category": draw(_category),
|
||||
}
|
||||
|
||||
|
||||
# Generate a non-empty list of artifact specs
|
||||
_artifact_set = st.lists(artifact_spec(), min_size=1, max_size=15)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_test_artifact(folder: Path, spec: dict) -> Path:
|
||||
"""Write a minimal artifact file from spec to the given folder."""
|
||||
subfolder = folder / spec["category"]
|
||||
subfolder.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
file_path = subfolder / spec["filename"]
|
||||
|
||||
tags_str = ", ".join(spec["tags"]) if spec["tags"] else ""
|
||||
shareable_str = "true" if spec["shareable"] else "false"
|
||||
|
||||
frontmatter = f"""---
|
||||
type: meeting
|
||||
title: "{spec['title']}"
|
||||
tags: [{tags_str}]
|
||||
source_context: bahn
|
||||
created: 2025-07-01
|
||||
shareable: {shareable_str}
|
||||
---
|
||||
# {spec['title']}
|
||||
|
||||
Some content for testing.
|
||||
"""
|
||||
file_path.write_text(frontmatter, encoding="utf-8")
|
||||
return file_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given(artifacts=_artifact_set)
|
||||
@settings(max_examples=100, deadline=None)
|
||||
def test_federation_sync_excludes_all_non_shareable(
|
||||
artifacts: list[dict], tmp_path_factory
|
||||
) -> None:
|
||||
"""**Validates: Requirements 14.4**
|
||||
|
||||
Property 22: For any generated set of artifacts with varying shareable values,
|
||||
the filter always excludes ALL shareable: false and includes ALL shareable: true
|
||||
(assuming no other exclusion criteria like cross-context links or ctx-guard).
|
||||
"""
|
||||
# Create a unique tmp directory for this test case
|
||||
tmp_path = tmp_path_factory.mktemp("fed_sync")
|
||||
knowledge_path = tmp_path / "bahn" / "knowledge"
|
||||
knowledge_path.mkdir(parents=True)
|
||||
|
||||
# De-duplicate filenames within same category to avoid overwrites
|
||||
written: dict[str, tuple[Path, bool]] = {}
|
||||
for spec in artifacts:
|
||||
key = f"{spec['category']}/{spec['filename']}"
|
||||
if key in written:
|
||||
continue
|
||||
file_path = _write_test_artifact(knowledge_path, spec)
|
||||
written[key] = (file_path, spec["shareable"])
|
||||
|
||||
# Run the filter (no ctx_guard to isolate shareable logic only)
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
ctx_guard=None,
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(knowledge_path)
|
||||
|
||||
# Partition expected results
|
||||
expected_included = {path for path, shareable in written.values() if shareable}
|
||||
expected_excluded = {path for path, shareable in written.values() if not shareable}
|
||||
|
||||
# Property assertions:
|
||||
# 1. ALL shareable: false artifacts MUST be excluded
|
||||
actual_included_set = set(result.included)
|
||||
for path in expected_excluded:
|
||||
assert path not in actual_included_set, (
|
||||
f"Artifact with shareable: false was included: {path.name}"
|
||||
)
|
||||
|
||||
# 2. ALL shareable: true artifacts MUST be included
|
||||
actual_excluded_paths = {exc.path for exc in result.excluded}
|
||||
for path in expected_included:
|
||||
assert path in actual_included_set, (
|
||||
f"Artifact with shareable: true was not included: {path.name}"
|
||||
)
|
||||
assert path not in actual_excluded_paths, (
|
||||
f"Artifact with shareable: true was excluded: {path.name}"
|
||||
)
|
||||
|
||||
# 3. Excluded artifacts must all have reason "not_shareable"
|
||||
for exclusion in result.excluded:
|
||||
assert exclusion.reason == "not_shareable", (
|
||||
f"Unexpected exclusion reason: {exclusion.reason} for {exclusion.path.name}"
|
||||
)
|
||||
|
||||
# 4. Total count is consistent
|
||||
assert result.total == len(written)
|
||||
@@ -0,0 +1,330 @@
|
||||
"""Unit-Tests für den Federation-Sync-Filter.
|
||||
|
||||
Testet die Knowledge-Artefakt-Filterung bei der Federation-Synchronisation:
|
||||
- Artefakte mit shareable: false werden ausgeschlossen
|
||||
- ctx-guard-sensible Artefakte werden ausgeschlossen
|
||||
- Artefakte mit kontextübergreifenden Links werden ausgeschlossen
|
||||
- Artefakte mit shareable: true ohne Probleme werden eingeschlossen
|
||||
|
||||
Requirements: 14.1, 14.2, 14.3, 14.4, 14.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.integrations.federation import (
|
||||
FederationSyncFilter,
|
||||
FederationSyncFilterResult,
|
||||
SyncExclusion,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_knowledge(tmp_path: Path) -> Path:
|
||||
"""Creates a temporary knowledge folder structure."""
|
||||
knowledge_path = tmp_path / "bahn" / "knowledge"
|
||||
knowledge_path.mkdir(parents=True)
|
||||
(knowledge_path / "meetings").mkdir()
|
||||
(knowledge_path / "decisions").mkdir()
|
||||
(knowledge_path / "inbox").mkdir()
|
||||
return knowledge_path
|
||||
|
||||
|
||||
def _write_artifact(path: Path, shareable: bool = True, extra_frontmatter: str = "", content: str = "# Test\n\nSome content.") -> Path:
|
||||
"""Helper to write a minimal artifact file."""
|
||||
fm = f"""---
|
||||
type: meeting
|
||||
title: "Test Artifact"
|
||||
tags: [test]
|
||||
source_context: bahn
|
||||
created: 2025-07-01
|
||||
shareable: {str(shareable).lower()}
|
||||
{extra_frontmatter}---
|
||||
{content}"""
|
||||
path.write_text(fm, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
class TestFederationSyncFilterShareable:
|
||||
"""Tests for shareable field filtering (Req 14.4)."""
|
||||
|
||||
def test_excludes_artifact_with_shareable_false(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Artefakte mit shareable: false werden von der Sync ausgeschlossen."""
|
||||
artifact_path = tmp_knowledge / "meetings" / "private-meeting.md"
|
||||
_write_artifact(artifact_path, shareable=False)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.excluded) == 1
|
||||
assert result.excluded[0].reason == "not_shareable"
|
||||
assert artifact_path not in result.included
|
||||
|
||||
def test_includes_artifact_with_shareable_true(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Artefakte mit shareable: true werden synchronisiert."""
|
||||
artifact_path = tmp_knowledge / "meetings" / "team-meeting.md"
|
||||
_write_artifact(artifact_path, shareable=True)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.included) == 1
|
||||
assert artifact_path in result.included
|
||||
assert len(result.excluded) == 0
|
||||
|
||||
def test_mixed_shareable_artifacts(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Gemischte Artefakte: nur shareable: true werden eingeschlossen."""
|
||||
shareable_path = tmp_knowledge / "meetings" / "public.md"
|
||||
_write_artifact(shareable_path, shareable=True)
|
||||
|
||||
private_path = tmp_knowledge / "meetings" / "private.md"
|
||||
_write_artifact(private_path, shareable=False)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.included) == 1
|
||||
assert shareable_path in result.included
|
||||
assert len(result.excluded) == 1
|
||||
assert result.excluded[0].path == private_path
|
||||
|
||||
|
||||
class TestFederationSyncFilterCtxGuard:
|
||||
"""Tests for ctx-guard sensitivity filtering (Req 14.5)."""
|
||||
|
||||
def test_excludes_ctx_guard_sensitive_artifact(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""ctx-guard-geschützte Artefakte werden ausgeschlossen."""
|
||||
artifact_path = tmp_knowledge / "decisions" / "sensitive-decision.md"
|
||||
_write_artifact(artifact_path, shareable=True)
|
||||
|
||||
# Mock ctx_guard that denies access from shared context
|
||||
mock_guard = MagicMock()
|
||||
mock_guard.check_access.return_value = False
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
ctx_guard=mock_guard,
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.excluded) == 1
|
||||
assert result.excluded[0].reason == "ctx_guard_sensitive"
|
||||
|
||||
def test_includes_non_sensitive_artifact_with_ctx_guard(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Nicht-sensible Artefakte passieren den ctx-guard-Check."""
|
||||
artifact_path = tmp_knowledge / "decisions" / "public-decision.md"
|
||||
_write_artifact(artifact_path, shareable=True)
|
||||
|
||||
# Mock ctx_guard that allows access
|
||||
mock_guard = MagicMock()
|
||||
mock_guard.check_access.return_value = True
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
ctx_guard=mock_guard,
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.included) == 1
|
||||
assert artifact_path in result.included
|
||||
|
||||
def test_no_ctx_guard_skips_sensitivity_check(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Ohne ctx-guard wird die Sensitivitätsprüfung übersprungen."""
|
||||
artifact_path = tmp_knowledge / "decisions" / "some-decision.md"
|
||||
_write_artifact(artifact_path, shareable=True)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
ctx_guard=None,
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.included) == 1
|
||||
|
||||
|
||||
class TestFederationSyncFilterCrossContext:
|
||||
"""Tests for cross-context link filtering (Req 14.3)."""
|
||||
|
||||
def test_excludes_artifact_with_cross_context_link_in_frontmatter(
|
||||
self, tmp_knowledge: Path, tmp_path: Path
|
||||
):
|
||||
"""Artefakte mit Links auf andere Kontexte im Frontmatter werden ausgeschlossen."""
|
||||
artifact_path = tmp_knowledge / "meetings" / "cross-link.md"
|
||||
extra = 'links:\n - target: "dhive/knowledge/projects/some-project"\n relation: "references"\n'
|
||||
_write_artifact(artifact_path, shareable=True, extra_frontmatter=extra)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.excluded) == 1
|
||||
assert result.excluded[0].reason == "cross_context_links"
|
||||
|
||||
def test_excludes_artifact_with_wiki_link_to_other_context(
|
||||
self, tmp_knowledge: Path, tmp_path: Path
|
||||
):
|
||||
"""Artefakte mit Wiki-Links auf andere Kontexte im Content werden ausgeschlossen."""
|
||||
artifact_path = tmp_knowledge / "meetings" / "wiki-cross.md"
|
||||
content = "# Meeting\n\nSiehe auch [[dhive/knowledge/projects/other-project]] für Details."
|
||||
_write_artifact(artifact_path, shareable=True, content=content)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.excluded) == 1
|
||||
assert result.excluded[0].reason == "cross_context_links"
|
||||
|
||||
def test_allows_same_context_links(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Links innerhalb des eigenen Kontexts sind erlaubt."""
|
||||
artifact_path = tmp_knowledge / "meetings" / "same-context.md"
|
||||
extra = 'links:\n - target: "bahn/knowledge/decisions/api-redesign"\n relation: "discussed_in"\n'
|
||||
_write_artifact(artifact_path, shareable=True, extra_frontmatter=extra)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
# Same-context links should not be excluded
|
||||
assert len(result.included) == 1
|
||||
|
||||
def test_allows_relative_links_without_context(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Relative Links ohne Kontext-Präfix sind erlaubt."""
|
||||
artifact_path = tmp_knowledge / "meetings" / "relative-link.md"
|
||||
extra = 'links:\n - target: "decisions/api-redesign"\n relation: "references"\n'
|
||||
_write_artifact(artifact_path, shareable=True, extra_frontmatter=extra)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.included) == 1
|
||||
|
||||
|
||||
class TestFederationSyncFilterEdgeCases:
|
||||
"""Tests for edge cases and special scenarios."""
|
||||
|
||||
def test_empty_knowledge_folder(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Leerer Knowledge-Ordner ergibt leere Ergebnisse."""
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert len(result.included) == 0
|
||||
assert len(result.excluded) == 0
|
||||
|
||||
def test_nonexistent_knowledge_folder(self, tmp_path: Path):
|
||||
"""Nicht existierender Knowledge-Ordner ergibt leere Ergebnisse."""
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
nonexistent = tmp_path / "bahn" / "knowledge"
|
||||
result = sync_filter.filter_knowledge_artifacts(nonexistent)
|
||||
|
||||
assert len(result.included) == 0
|
||||
assert len(result.excluded) == 0
|
||||
|
||||
def test_markdown_without_frontmatter_passes(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Markdown-Dateien ohne Frontmatter werden durchgelassen (z.B. index.md)."""
|
||||
index_path = tmp_knowledge / "meetings" / "index.md"
|
||||
index_path.write_text("# Meetings\n\n- Meeting 1\n- Meeting 2\n", encoding="utf-8")
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
# Files without frontmatter pass through (they're not artifacts)
|
||||
assert index_path in result.included
|
||||
|
||||
def test_non_markdown_files_pass(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""Nicht-Markdown-Dateien werden nicht gefiltert (rglob nur *.md)."""
|
||||
yaml_path = tmp_knowledge / "_index.yaml"
|
||||
yaml_path.write_text("version: '1.0'\nartifacts: []\n", encoding="utf-8")
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
# YAML files are not scanned (only *.md)
|
||||
assert yaml_path not in result.included
|
||||
assert len(result.excluded) == 0
|
||||
|
||||
def test_should_sync_convenience_method(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""should_sync gibt True/False korrekt zurück."""
|
||||
shareable_path = tmp_knowledge / "meetings" / "public.md"
|
||||
_write_artifact(shareable_path, shareable=True)
|
||||
|
||||
private_path = tmp_knowledge / "meetings" / "private.md"
|
||||
_write_artifact(private_path, shareable=False)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
|
||||
assert sync_filter.should_sync(shareable_path) is True
|
||||
assert sync_filter.should_sync(private_path) is False
|
||||
|
||||
def test_get_sync_file_list(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""get_sync_file_list gibt nur synchronisierbare Pfade zurück."""
|
||||
shareable_path = tmp_knowledge / "meetings" / "public.md"
|
||||
_write_artifact(shareable_path, shareable=True)
|
||||
|
||||
private_path = tmp_knowledge / "meetings" / "private.md"
|
||||
_write_artifact(private_path, shareable=False)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
file_list = sync_filter.get_sync_file_list(tmp_knowledge)
|
||||
|
||||
assert shareable_path in file_list
|
||||
assert private_path not in file_list
|
||||
|
||||
def test_total_property(self, tmp_knowledge: Path, tmp_path: Path):
|
||||
"""total zählt included + excluded korrekt."""
|
||||
_write_artifact(tmp_knowledge / "meetings" / "a.md", shareable=True)
|
||||
_write_artifact(tmp_knowledge / "meetings" / "b.md", shareable=False)
|
||||
_write_artifact(tmp_knowledge / "meetings" / "c.md", shareable=True)
|
||||
|
||||
sync_filter = FederationSyncFilter(
|
||||
monorepo_root=tmp_path,
|
||||
context="bahn",
|
||||
)
|
||||
result = sync_filter.filter_knowledge_artifacts(tmp_knowledge)
|
||||
|
||||
assert result.total == 3
|
||||
assert len(result.included) == 2
|
||||
assert len(result.excluded) == 1
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Property-Based Tests für File Extension Format Detection.
|
||||
|
||||
**Validates: Requirements 4.7**
|
||||
|
||||
Property 9: File Extension Format Detection
|
||||
*For any* file path with a supported extension (`.md`, `.txt`, `.pdf`, `.png`,
|
||||
`.jpg`, `.jpeg`, `.docx`), the format detection function SHALL return the correct
|
||||
format type. For unsupported extensions, it SHALL raise an appropriate error.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, assume
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.sources.base import (
|
||||
EXTENSION_FORMAT_MAP,
|
||||
SUPPORTED_EXTENSIONS,
|
||||
detect_format,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Supported extensions sampled from the canonical map
|
||||
supported_ext_st = st.sampled_from(list(EXTENSION_FORMAT_MAP.keys()))
|
||||
|
||||
# Path prefix segments (directory components, no dots allowed to avoid
|
||||
# accidental extension-like suffixes)
|
||||
_dir_segment_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Lu", "Nd"),
|
||||
whitelist_characters="-_",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=20,
|
||||
)
|
||||
|
||||
# File basename (stem) without extension – at least one char, no dots
|
||||
_file_stem_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Lu", "Nd"),
|
||||
whitelist_characters="-_ ",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=30,
|
||||
).filter(lambda s: s.strip() != "")
|
||||
|
||||
# Directory depth: 0 to 4 levels
|
||||
_dir_depth_st = st.integers(min_value=0, max_value=4)
|
||||
|
||||
# Path separator styles
|
||||
_separator_st = st.sampled_from(["/", "\\"])
|
||||
|
||||
# Unsupported extensions: alphabetic strings NOT in the supported set
|
||||
unsupported_ext_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("Ll",)),
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
).filter(lambda s: s not in SUPPORTED_EXTENSIONS)
|
||||
|
||||
# Case variations for extensions (mixed case)
|
||||
_case_variant_st = st.sampled_from(["lower", "upper", "title", "mixed"])
|
||||
|
||||
|
||||
def apply_case(ext: str, variant: str) -> str:
|
||||
"""Apply a case variant to an extension string."""
|
||||
if variant == "lower":
|
||||
return ext.lower()
|
||||
elif variant == "upper":
|
||||
return ext.upper()
|
||||
elif variant == "title":
|
||||
return ext.title()
|
||||
else: # mixed
|
||||
return "".join(
|
||||
c.upper() if i % 2 == 0 else c.lower()
|
||||
for i, c in enumerate(ext)
|
||||
)
|
||||
|
||||
|
||||
# Composite strategy: generate a full file path with a supported extension
|
||||
@st.composite
|
||||
def supported_file_path_st(draw: st.DrawFn) -> tuple[str, str]:
|
||||
"""Generate a file path with a supported extension, return (path, ext)."""
|
||||
ext = draw(supported_ext_st)
|
||||
stem = draw(_file_stem_st)
|
||||
depth = draw(_dir_depth_st)
|
||||
sep = draw(_separator_st)
|
||||
|
||||
parts: list[str] = []
|
||||
for _ in range(depth):
|
||||
parts.append(draw(_dir_segment_st))
|
||||
parts.append(f"{stem}.{ext}")
|
||||
|
||||
path = sep.join(parts)
|
||||
return path, ext
|
||||
|
||||
|
||||
@st.composite
|
||||
def unsupported_file_path_st(draw: st.DrawFn) -> str:
|
||||
"""Generate a file path with an unsupported extension."""
|
||||
ext = draw(unsupported_ext_st)
|
||||
stem = draw(_file_stem_st)
|
||||
depth = draw(_dir_depth_st)
|
||||
sep = draw(_separator_st)
|
||||
|
||||
parts: list[str] = []
|
||||
for _ in range(depth):
|
||||
parts.append(draw(_dir_segment_st))
|
||||
parts.append(f"{stem}.{ext}")
|
||||
|
||||
return sep.join(parts)
|
||||
|
||||
|
||||
@st.composite
|
||||
def case_insensitive_path_st(draw: st.DrawFn) -> tuple[str, str]:
|
||||
"""Generate path with case-varied extension, return (path, expected_format)."""
|
||||
ext = draw(supported_ext_st)
|
||||
case_var = draw(_case_variant_st)
|
||||
varied_ext = apply_case(ext, case_var)
|
||||
stem = draw(_file_stem_st)
|
||||
depth = draw(_dir_depth_st)
|
||||
sep = draw(_separator_st)
|
||||
|
||||
parts: list[str] = []
|
||||
for _ in range(depth):
|
||||
parts.append(draw(_dir_segment_st))
|
||||
parts.append(f"{stem}.{varied_ext}")
|
||||
|
||||
path = sep.join(parts)
|
||||
expected_format = EXTENSION_FORMAT_MAP[ext]
|
||||
return path, expected_format
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 9: File Extension Format Detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFileExtensionFormatDetection:
|
||||
"""**Validates: Requirements 4.7**
|
||||
|
||||
Property 9: For any file path with a supported extension, detect_format
|
||||
SHALL return the correct format type. For unsupported extensions, it SHALL
|
||||
raise ValueError.
|
||||
"""
|
||||
|
||||
@given(data=supported_file_path_st())
|
||||
@settings(max_examples=300)
|
||||
def test_supported_extension_returns_correct_format(
|
||||
self, data: tuple[str, str]
|
||||
) -> None:
|
||||
"""Any path with a supported extension maps to the correct format type."""
|
||||
path, ext = data
|
||||
result = detect_format(path)
|
||||
expected = EXTENSION_FORMAT_MAP[ext]
|
||||
assert result == expected, (
|
||||
f"detect_format({path!r}) returned {result!r}, "
|
||||
f"expected {expected!r} for extension '.{ext}'"
|
||||
)
|
||||
|
||||
@given(path=unsupported_file_path_st())
|
||||
@settings(max_examples=300)
|
||||
def test_unsupported_extension_raises_valueerror(self, path: str) -> None:
|
||||
"""Any path with an unsupported extension raises ValueError."""
|
||||
with pytest.raises(ValueError):
|
||||
detect_format(path)
|
||||
|
||||
@given(data=case_insensitive_path_st())
|
||||
@settings(max_examples=300)
|
||||
def test_case_insensitive_extension_detection(
|
||||
self, data: tuple[str, str]
|
||||
) -> None:
|
||||
"""Extension detection is case-insensitive (e.g., .MD, .Pdf, .JPG)."""
|
||||
path, expected_format = data
|
||||
result = detect_format(path)
|
||||
assert result == expected_format, (
|
||||
f"detect_format({path!r}) returned {result!r}, "
|
||||
f"expected {expected_format!r} (case-insensitive)"
|
||||
)
|
||||
|
||||
@given(ext=supported_ext_st)
|
||||
@settings(max_examples=100)
|
||||
def test_format_type_is_never_empty(self, ext: str) -> None:
|
||||
"""The returned format type is never an empty string."""
|
||||
result = detect_format(f"file.{ext}")
|
||||
assert result != ""
|
||||
assert isinstance(result, str)
|
||||
|
||||
@given(ext=supported_ext_st)
|
||||
@settings(max_examples=100)
|
||||
def test_format_type_is_lowercase(self, ext: str) -> None:
|
||||
"""The returned format type is always lowercase."""
|
||||
result = detect_format(f"file.{ext}")
|
||||
assert result == result.lower()
|
||||
|
||||
def test_empty_path_raises_valueerror(self) -> None:
|
||||
"""An empty file path raises ValueError."""
|
||||
with pytest.raises(ValueError):
|
||||
detect_format("")
|
||||
|
||||
def test_whitespace_only_path_raises_valueerror(self) -> None:
|
||||
"""A whitespace-only file path raises ValueError."""
|
||||
with pytest.raises(ValueError):
|
||||
detect_format(" ")
|
||||
|
||||
def test_no_extension_raises_valueerror(self) -> None:
|
||||
"""A path without any extension raises ValueError."""
|
||||
with pytest.raises(ValueError):
|
||||
detect_format("readme")
|
||||
|
||||
def test_dotfile_without_extension_raises_valueerror(self) -> None:
|
||||
"""A dotfile like .gitignore (no real extension) raises ValueError."""
|
||||
with pytest.raises(ValueError):
|
||||
detect_format(".gitignore")
|
||||
|
||||
def test_all_supported_extensions_covered(self) -> None:
|
||||
"""Verify all seven required extensions are in the map."""
|
||||
required = {"md", "txt", "pdf", "png", "jpg", "jpeg", "docx"}
|
||||
assert required == SUPPORTED_EXTENSIONS
|
||||
|
||||
def test_image_extensions_all_map_to_image(self) -> None:
|
||||
"""All image extensions (.png, .jpg, .jpeg) map to 'image'."""
|
||||
for ext in ("png", "jpg", "jpeg"):
|
||||
assert detect_format(f"photo.{ext}") == "image"
|
||||
|
||||
def test_specific_format_values(self) -> None:
|
||||
"""Verify the exact mapping for each supported extension."""
|
||||
assert detect_format("notes.md") == "markdown"
|
||||
assert detect_format("readme.txt") == "text"
|
||||
assert detect_format("document.pdf") == "pdf"
|
||||
assert detect_format("screenshot.png") == "image"
|
||||
assert detect_format("photo.jpg") == "image"
|
||||
assert detect_format("photo.jpeg") == "image"
|
||||
assert detect_format("report.docx") == "docx"
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Property-Based Tests für Git Commit Message Format.
|
||||
|
||||
**Validates: Requirements 16.2**
|
||||
|
||||
Property 23: Git Commit Message Format
|
||||
*For any* valid combination of context (bahn|dhive|privat), action (ingest|capture),
|
||||
count (positive integer), and source name, the generated commit message SHALL match
|
||||
the pattern `knowledge({context}): {action} {count} artifacts from {source}`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.ingestion.git_integration import generate_commit_message
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Valid contexts as defined in the spec
|
||||
context_st = st.sampled_from(["bahn", "dhive", "privat"])
|
||||
|
||||
# Valid actions as defined in the spec
|
||||
action_st = st.sampled_from(["ingest", "capture"])
|
||||
|
||||
# Positive integer count (at least 1 artifact)
|
||||
count_st = st.integers(min_value=1, max_value=10000)
|
||||
|
||||
# Source names: non-empty strings representing source identifiers
|
||||
# (confluence, jira, inbox, email, markdown, etc.)
|
||||
source_name_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N"),
|
||||
whitelist_characters="-_",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=50,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Commit message pattern
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
COMMIT_MSG_PATTERN = re.compile(
|
||||
r"^knowledge\((bahn|dhive|privat)\): (ingest|capture) (\d+) artifacts from (.+)$"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property: Commit Message Format Matches Pattern
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGitCommitMessageFormatProperty:
|
||||
"""**Validates: Requirements 16.2**"""
|
||||
|
||||
@given(
|
||||
context=context_st,
|
||||
action=action_st,
|
||||
count=count_st,
|
||||
source=source_name_st,
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_commit_message_matches_pattern(
|
||||
self,
|
||||
context: str,
|
||||
action: str,
|
||||
count: int,
|
||||
source: str,
|
||||
) -> None:
|
||||
"""For any valid inputs, the commit message matches the specified pattern."""
|
||||
message = generate_commit_message(context, action, count, source)
|
||||
match = COMMIT_MSG_PATTERN.match(message)
|
||||
assert match is not None, (
|
||||
f"Commit message does not match expected pattern.\n"
|
||||
f" Generated: {message!r}\n"
|
||||
f" Pattern: {COMMIT_MSG_PATTERN.pattern}"
|
||||
)
|
||||
|
||||
@given(
|
||||
context=context_st,
|
||||
action=action_st,
|
||||
count=count_st,
|
||||
source=source_name_st,
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_commit_message_preserves_inputs(
|
||||
self,
|
||||
context: str,
|
||||
action: str,
|
||||
count: int,
|
||||
source: str,
|
||||
) -> None:
|
||||
"""The commit message contains the exact input values in correct positions."""
|
||||
message = generate_commit_message(context, action, count, source)
|
||||
match = COMMIT_MSG_PATTERN.match(message)
|
||||
assert match is not None
|
||||
|
||||
# Verify each captured group matches the input
|
||||
assert match.group(1) == context, f"Context mismatch: {match.group(1)} != {context}"
|
||||
assert match.group(2) == action, f"Action mismatch: {match.group(2)} != {action}"
|
||||
assert match.group(3) == str(count), f"Count mismatch: {match.group(3)} != {count}"
|
||||
assert match.group(4) == source, f"Source mismatch: {match.group(4)} != {source}"
|
||||
|
||||
@given(
|
||||
context=context_st,
|
||||
action=action_st,
|
||||
count=count_st,
|
||||
source=source_name_st,
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_commit_message_exact_format(
|
||||
self,
|
||||
context: str,
|
||||
action: str,
|
||||
count: int,
|
||||
source: str,
|
||||
) -> None:
|
||||
"""The commit message exactly equals the expected formatted string."""
|
||||
message = generate_commit_message(context, action, count, source)
|
||||
expected = f"knowledge({context}): {action} {count} artifacts from {source}"
|
||||
assert message == expected
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Property-Based Tests für Image Source File Reference Preservation.
|
||||
|
||||
**Validates: Requirements 4.5**
|
||||
|
||||
Property 10: Image Source File Reference Preservation
|
||||
*For any* artifact created from an image source (PNG, JPG, JPEG), the resulting
|
||||
artifact's frontmatter SHALL contain a `source.file` field referencing the original
|
||||
image file path. The `source.type` field SHALL be "image" for image files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, assume, HealthCheck
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
from monorepo.knowledge.sources.pdf import (
|
||||
IMAGE_EXTENSIONS,
|
||||
PDFSource,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Image extensions without the dot (lowercase)
|
||||
_image_extensions = st.sampled_from(["png", "jpg", "jpeg"])
|
||||
|
||||
# File name stem: at least one char, no dots, safe filesystem characters
|
||||
_file_stem_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Lu", "Nd"),
|
||||
whitelist_characters="-_",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=30,
|
||||
).filter(lambda s: s.strip() != "")
|
||||
|
||||
# Directory depth: 0 to 3 levels
|
||||
_dir_depth_st = st.integers(min_value=0, max_value=3)
|
||||
|
||||
# Directory segment
|
||||
_dir_segment_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Lu", "Nd"),
|
||||
whitelist_characters="-_",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=15,
|
||||
)
|
||||
|
||||
# Context for the extraction
|
||||
_context_st = st.sampled_from(["bahn", "dhive", "privat"])
|
||||
|
||||
# OCR result text (can be empty or contain text)
|
||||
_ocr_text_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Z")),
|
||||
min_size=0,
|
||||
max_size=200,
|
||||
)
|
||||
|
||||
# Whether OCR fails
|
||||
_ocr_failed_st = st.booleans()
|
||||
|
||||
|
||||
@st.composite
|
||||
def image_file_scenario_st(draw: st.DrawFn) -> dict:
|
||||
"""Generate a complete scenario for image file processing.
|
||||
|
||||
Returns a dict with: stem, ext, depth, dir_segments, context, ocr_text, ocr_failed
|
||||
"""
|
||||
stem = draw(_file_stem_st)
|
||||
ext = draw(_image_extensions)
|
||||
depth = draw(_dir_depth_st)
|
||||
dir_segments = [draw(_dir_segment_st) for _ in range(depth)]
|
||||
context = draw(_context_st)
|
||||
ocr_text = draw(_ocr_text_st)
|
||||
ocr_failed = draw(_ocr_failed_st)
|
||||
|
||||
return {
|
||||
"stem": stem,
|
||||
"ext": ext,
|
||||
"depth": depth,
|
||||
"dir_segments": dir_segments,
|
||||
"context": context,
|
||||
"ocr_text": ocr_text,
|
||||
"ocr_failed": ocr_failed,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 10: Image Source File Reference Preservation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestImageSourceFileReferencePreservation:
|
||||
"""**Validates: Requirements 4.5**
|
||||
|
||||
Property 10: For any artifact created from an image source (PNG, JPG, JPEG),
|
||||
the resulting artifact's frontmatter SHALL contain a `source.file` field
|
||||
referencing the original image file path, and `source.type` SHALL be "image".
|
||||
"""
|
||||
|
||||
@given(scenario=image_file_scenario_st())
|
||||
@settings(max_examples=200, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_source_file_references_original_path(
|
||||
self, scenario: dict, tmp_path: Path
|
||||
) -> None:
|
||||
"""For any image file, source['file'] matches the original file path."""
|
||||
stem = scenario["stem"]
|
||||
ext = scenario["ext"]
|
||||
dir_segments = scenario["dir_segments"]
|
||||
context = scenario["context"]
|
||||
ocr_text = scenario["ocr_text"]
|
||||
ocr_failed = scenario["ocr_failed"]
|
||||
|
||||
# Build directory structure
|
||||
target_dir = tmp_path
|
||||
for seg in dir_segments:
|
||||
target_dir = target_dir / seg
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create the image file
|
||||
image_file = target_dir / f"{stem}.{ext}"
|
||||
image_file.write_bytes(b"\x89PNG\r\n\x1a\n dummy content")
|
||||
|
||||
# Configure and run
|
||||
source = PDFSource()
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test Image Source",
|
||||
params={"path": str(image_file)},
|
||||
)
|
||||
|
||||
with patch("monorepo.knowledge.sources.pdf._perform_ocr") as mock_ocr:
|
||||
mock_ocr.return_value = (ocr_text, ocr_failed)
|
||||
result = source.extract(config, context)
|
||||
|
||||
# Property assertion: exactly one artifact with correct source.file
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert "file" in artifact.metadata.source
|
||||
assert artifact.metadata.source["file"] == str(image_file)
|
||||
|
||||
@given(scenario=image_file_scenario_st())
|
||||
@settings(max_examples=200, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_source_type_is_image(
|
||||
self, scenario: dict, tmp_path: Path
|
||||
) -> None:
|
||||
"""For any image file, source['type'] is always 'image'."""
|
||||
stem = scenario["stem"]
|
||||
ext = scenario["ext"]
|
||||
context = scenario["context"]
|
||||
ocr_text = scenario["ocr_text"]
|
||||
ocr_failed = scenario["ocr_failed"]
|
||||
|
||||
# Create the image file directly in tmp_path
|
||||
image_file = tmp_path / f"{stem}.{ext}"
|
||||
image_file.write_bytes(b"\xff\xd8\xff\xe0 dummy image")
|
||||
|
||||
source = PDFSource()
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test Image Source",
|
||||
params={"path": str(image_file)},
|
||||
)
|
||||
|
||||
with patch("monorepo.knowledge.sources.pdf._perform_ocr") as mock_ocr:
|
||||
mock_ocr.return_value = (ocr_text, ocr_failed)
|
||||
result = source.extract(config, context)
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert "type" in artifact.metadata.source
|
||||
assert artifact.metadata.source["type"] == "image"
|
||||
|
||||
@given(scenario=image_file_scenario_st())
|
||||
@settings(max_examples=200, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_source_file_and_type_present_regardless_of_ocr_outcome(
|
||||
self, scenario: dict, tmp_path: Path
|
||||
) -> None:
|
||||
"""source.file and source.type are set regardless of OCR success/failure."""
|
||||
stem = scenario["stem"]
|
||||
ext = scenario["ext"]
|
||||
context = scenario["context"]
|
||||
ocr_text = scenario["ocr_text"]
|
||||
ocr_failed = scenario["ocr_failed"]
|
||||
|
||||
image_file = tmp_path / f"{stem}.{ext}"
|
||||
image_file.write_bytes(b"\x89PNG dummy")
|
||||
|
||||
source = PDFSource()
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test Image Source",
|
||||
params={"path": str(image_file)},
|
||||
)
|
||||
|
||||
with patch("monorepo.knowledge.sources.pdf._perform_ocr") as mock_ocr:
|
||||
mock_ocr.return_value = (ocr_text, ocr_failed)
|
||||
result = source.extract(config, context)
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
|
||||
# Both fields must always be present
|
||||
assert "file" in artifact.metadata.source
|
||||
assert "type" in artifact.metadata.source
|
||||
assert artifact.metadata.source["file"] == str(image_file)
|
||||
assert artifact.metadata.source["type"] == "image"
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Unit-Tests für monorepo.knowledge.ingestion.config.
|
||||
|
||||
Testet SourceConfig, PipelineSettings, Env-Var-Auflösung,
|
||||
Validierung und sources.yaml-Parsing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.ingestion.config import (
|
||||
ConfigValidationError,
|
||||
EnrichmentSettings,
|
||||
GitSettings,
|
||||
OCRSettings,
|
||||
PipelineSettings,
|
||||
SourceConfig,
|
||||
load_sources_config,
|
||||
resolve_env_vars,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SourceConfig Dataclass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSourceConfig:
|
||||
"""Tests für die SourceConfig-Dataclass."""
|
||||
|
||||
def test_required_fields(self) -> None:
|
||||
cfg = SourceConfig(type="confluence", name="Team Wiki")
|
||||
assert cfg.type == "confluence"
|
||||
assert cfg.name == "Team Wiki"
|
||||
|
||||
def test_defaults(self) -> None:
|
||||
cfg = SourceConfig(type="jira", name="Sprint Issues")
|
||||
assert cfg.params == {}
|
||||
assert cfg.target_folder == ""
|
||||
assert cfg.sync_frequency == "manual"
|
||||
assert cfg.enabled is True
|
||||
|
||||
def test_full_config(self) -> None:
|
||||
cfg = SourceConfig(
|
||||
type="confluence",
|
||||
name="DB Confluence",
|
||||
params={"base_url": "https://confluence.bahn.de", "spaces": ["ACV2"]},
|
||||
target_folder="references",
|
||||
sync_frequency="daily",
|
||||
enabled=True,
|
||||
)
|
||||
assert cfg.params["base_url"] == "https://confluence.bahn.de"
|
||||
assert cfg.target_folder == "references"
|
||||
assert cfg.sync_frequency == "daily"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PipelineSettings Dataclass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPipelineSettings:
|
||||
"""Tests für PipelineSettings und Unter-Dataclasses."""
|
||||
|
||||
def test_defaults(self) -> None:
|
||||
settings = PipelineSettings()
|
||||
assert settings.create_tasks is True
|
||||
assert settings.enrichment.provider == "kiro"
|
||||
assert settings.enrichment.confidence_threshold == 0.7
|
||||
assert settings.ocr.provider == "kiro"
|
||||
assert settings.ocr.languages == "deu+eng"
|
||||
assert settings.git.auto_commit is True
|
||||
|
||||
def test_custom_settings(self) -> None:
|
||||
settings = PipelineSettings(
|
||||
create_tasks=False,
|
||||
enrichment=EnrichmentSettings(provider="litellm", model="gpt-4"),
|
||||
ocr=OCRSettings(provider="tesseract", languages="deu"),
|
||||
git=GitSettings(auto_commit=False),
|
||||
)
|
||||
assert settings.create_tasks is False
|
||||
assert settings.enrichment.provider == "litellm"
|
||||
assert settings.enrichment.model == "gpt-4"
|
||||
assert settings.ocr.provider == "tesseract"
|
||||
assert settings.git.auto_commit is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_env_vars
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveEnvVars:
|
||||
"""Tests für die Umgebungsvariablen-Auflösung."""
|
||||
|
||||
def test_no_vars(self) -> None:
|
||||
assert resolve_env_vars("hello world") == "hello world"
|
||||
|
||||
def test_single_var(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("MY_TOKEN", "secret123")
|
||||
assert resolve_env_vars("${MY_TOKEN}") == "secret123"
|
||||
|
||||
def test_var_in_string(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HOST", "example.com")
|
||||
assert resolve_env_vars("https://${HOST}/api") == "https://example.com/api"
|
||||
|
||||
def test_multiple_vars(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("USER", "admin")
|
||||
monkeypatch.setenv("PASS", "secret")
|
||||
result = resolve_env_vars("${USER}:${PASS}")
|
||||
assert result == "admin:secret"
|
||||
|
||||
def test_unset_var_raises(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("NONEXISTENT_VAR_XYZ", raising=False)
|
||||
with pytest.raises(ConfigValidationError) as exc_info:
|
||||
resolve_env_vars("${NONEXISTENT_VAR_XYZ}")
|
||||
assert "NONEXISTENT_VAR_XYZ" in str(exc_info.value)
|
||||
|
||||
def test_empty_string(self) -> None:
|
||||
assert resolve_env_vars("") == ""
|
||||
|
||||
def test_var_with_empty_value(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("EMPTY_VAR", "")
|
||||
assert resolve_env_vars("prefix_${EMPTY_VAR}_suffix") == "prefix__suffix"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# load_sources_config – Erfolgsfälle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLoadSourcesConfig:
|
||||
"""Tests für load_sources_config."""
|
||||
|
||||
def test_basic_config(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("CONF_TOKEN", "tok123")
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources:
|
||||
- name: "Team Confluence"
|
||||
type: confluence
|
||||
enabled: true
|
||||
sync_frequency: daily
|
||||
params:
|
||||
base_url: "https://confluence.bahn.de"
|
||||
api_key: "${CONF_TOKEN}"
|
||||
spaces: ["ACV2"]
|
||||
target_folder: references
|
||||
settings:
|
||||
create_tasks: true
|
||||
enrichment:
|
||||
provider: "kiro"
|
||||
confidence_threshold: 0.8
|
||||
ocr:
|
||||
provider: "kiro"
|
||||
git:
|
||||
auto_commit: false
|
||||
"""), encoding="utf-8")
|
||||
|
||||
sources, settings = load_sources_config(config_file)
|
||||
|
||||
assert len(sources) == 1
|
||||
src = sources[0]
|
||||
assert src.type == "confluence"
|
||||
assert src.name == "Team Confluence"
|
||||
assert src.params["api_key"] == "tok123"
|
||||
assert src.params["spaces"] == ["ACV2"]
|
||||
assert src.target_folder == "references"
|
||||
assert src.sync_frequency == "daily"
|
||||
assert src.enabled is True
|
||||
|
||||
assert settings.create_tasks is True
|
||||
assert settings.enrichment.provider == "kiro"
|
||||
assert settings.enrichment.confidence_threshold == 0.8
|
||||
assert settings.git.auto_commit is False
|
||||
|
||||
def test_multiple_sources(self, tmp_path: Path) -> None:
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources:
|
||||
- name: "Source A"
|
||||
type: file
|
||||
params:
|
||||
path: "/data"
|
||||
- name: "Source B"
|
||||
type: jira
|
||||
enabled: false
|
||||
params:
|
||||
jql: "project = X"
|
||||
"""), encoding="utf-8")
|
||||
|
||||
sources, settings = load_sources_config(config_file)
|
||||
assert len(sources) == 2
|
||||
assert sources[0].name == "Source A"
|
||||
assert sources[1].name == "Source B"
|
||||
assert sources[1].enabled is False
|
||||
|
||||
def test_empty_sources_list(self, tmp_path: Path) -> None:
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources: []
|
||||
"""), encoding="utf-8")
|
||||
|
||||
sources, settings = load_sources_config(config_file)
|
||||
assert sources == []
|
||||
assert isinstance(settings, PipelineSettings)
|
||||
|
||||
def test_no_settings_block(self, tmp_path: Path) -> None:
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources:
|
||||
- name: "Minimal"
|
||||
type: file
|
||||
"""), encoding="utf-8")
|
||||
|
||||
sources, settings = load_sources_config(config_file)
|
||||
assert len(sources) == 1
|
||||
# Should use defaults
|
||||
assert settings.create_tasks is True
|
||||
assert settings.enrichment.provider == "kiro"
|
||||
|
||||
def test_file_not_found(self, tmp_path: Path) -> None:
|
||||
with pytest.raises(FileNotFoundError):
|
||||
load_sources_config(tmp_path / "nonexistent.yaml")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# load_sources_config – Validierungsfehler
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLoadSourcesConfigValidation:
|
||||
"""Tests für Validierungsfehler in load_sources_config."""
|
||||
|
||||
def test_missing_type(self, tmp_path: Path) -> None:
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources:
|
||||
- name: "No Type"
|
||||
params: {}
|
||||
"""), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ConfigValidationError) as exc_info:
|
||||
load_sources_config(config_file)
|
||||
assert "type" in str(exc_info.value).lower()
|
||||
|
||||
def test_missing_name(self, tmp_path: Path) -> None:
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources:
|
||||
- type: confluence
|
||||
params: {}
|
||||
"""), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ConfigValidationError) as exc_info:
|
||||
load_sources_config(config_file)
|
||||
assert "name" in str(exc_info.value).lower()
|
||||
|
||||
def test_unresolvable_env_var(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("MISSING_SECRET_XYZ", raising=False)
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources:
|
||||
- name: "Bad Source"
|
||||
type: confluence
|
||||
params:
|
||||
api_key: "${MISSING_SECRET_XYZ}"
|
||||
"""), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ConfigValidationError) as exc_info:
|
||||
load_sources_config(config_file)
|
||||
assert "MISSING_SECRET_XYZ" in str(exc_info.value)
|
||||
|
||||
def test_multiple_errors_collected(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("TOKEN_A", raising=False)
|
||||
monkeypatch.delenv("TOKEN_B", raising=False)
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources:
|
||||
- name: "Source 1"
|
||||
type: confluence
|
||||
params:
|
||||
key_a: "${TOKEN_A}"
|
||||
key_b: "${TOKEN_B}"
|
||||
"""), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ConfigValidationError) as exc_info:
|
||||
load_sources_config(config_file)
|
||||
assert "TOKEN_A" in str(exc_info.value)
|
||||
assert "TOKEN_B" in str(exc_info.value)
|
||||
|
||||
def test_missing_fields_and_env_vars(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("GHOST_VAR", raising=False)
|
||||
config_file = tmp_path / "sources.yaml"
|
||||
config_file.write_text(dedent("""\
|
||||
version: "1.0"
|
||||
sources:
|
||||
- type: jira
|
||||
params:
|
||||
token: "${GHOST_VAR}"
|
||||
"""), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ConfigValidationError) as exc_info:
|
||||
load_sources_config(config_file)
|
||||
# Both missing name and unresolvable env var
|
||||
assert "name" in str(exc_info.value).lower()
|
||||
assert "GHOST_VAR" in str(exc_info.value)
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Tests for the ContextRouter module."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.ingestion.config import SourceConfig
|
||||
from monorepo.knowledge.ingestion.router import (
|
||||
CATEGORY_FOLDER_MAP,
|
||||
DEFAULT_FOLDER,
|
||||
ContextRouter,
|
||||
RoutingDecision,
|
||||
category_to_folder,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# category_to_folder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCategoryToFolder:
|
||||
"""Tests for the category_to_folder helper."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"category,expected",
|
||||
[
|
||||
("meeting", "meetings"),
|
||||
("decision", "decisions"),
|
||||
("project", "projects"),
|
||||
("reference", "references"),
|
||||
("link", "links"),
|
||||
("inbox", "inbox"),
|
||||
],
|
||||
)
|
||||
def test_known_categories(self, category: str, expected: str) -> None:
|
||||
assert category_to_folder(category) == expected
|
||||
|
||||
def test_unknown_category_defaults_to_inbox(self) -> None:
|
||||
assert category_to_folder("unknown") == "inbox"
|
||||
assert category_to_folder("random") == "inbox"
|
||||
|
||||
def test_case_insensitive(self) -> None:
|
||||
assert category_to_folder("Meeting") == "meetings"
|
||||
assert category_to_folder("DECISION") == "decisions"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ContextRouter.determine_context
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDetermineContext:
|
||||
"""Tests for ContextRouter.determine_context."""
|
||||
|
||||
def setup_method(self) -> None:
|
||||
self.router = ContextRouter(Path("/monorepo"))
|
||||
|
||||
def test_from_source_config_params(self) -> None:
|
||||
config = SourceConfig(
|
||||
type="confluence",
|
||||
name="Test",
|
||||
params={"context": "bahn"},
|
||||
)
|
||||
assert self.router.determine_context(source_config=config) == "bahn"
|
||||
|
||||
def test_from_source_config_dhive(self) -> None:
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test",
|
||||
params={"context": "dhive"},
|
||||
)
|
||||
assert self.router.determine_context(source_config=config) == "dhive"
|
||||
|
||||
def test_from_source_config_privat(self) -> None:
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test",
|
||||
params={"context": "privat"},
|
||||
)
|
||||
assert self.router.determine_context(source_config=config) == "privat"
|
||||
|
||||
def test_from_cwd_bahn(self) -> None:
|
||||
cwd = Path("/monorepo/bahn/wissensdatenbank")
|
||||
assert self.router.determine_context(cwd=cwd) == "bahn"
|
||||
|
||||
def test_from_cwd_dhive(self) -> None:
|
||||
cwd = Path("/monorepo/dhive/projects/kiq")
|
||||
assert self.router.determine_context(cwd=cwd) == "dhive"
|
||||
|
||||
def test_from_cwd_privat(self) -> None:
|
||||
cwd = Path("/monorepo/privat/knowledge/meetings")
|
||||
assert self.router.determine_context(cwd=cwd) == "privat"
|
||||
|
||||
def test_from_cwd_windows_paths(self) -> None:
|
||||
cwd = Path("C:\\Users\\user\\monorepo\\bahn\\something")
|
||||
assert self.router.determine_context(cwd=cwd) == "bahn"
|
||||
|
||||
def test_source_config_takes_precedence_over_cwd(self) -> None:
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test",
|
||||
params={"context": "dhive"},
|
||||
)
|
||||
cwd = Path("/monorepo/bahn/something")
|
||||
assert self.router.determine_context(source_config=config, cwd=cwd) == "dhive"
|
||||
|
||||
def test_falls_back_to_cwd_when_config_has_no_context(self) -> None:
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test",
|
||||
params={"url": "http://example.com"},
|
||||
)
|
||||
cwd = Path("/monorepo/privat/notes")
|
||||
assert self.router.determine_context(source_config=config, cwd=cwd) == "privat"
|
||||
|
||||
def test_raises_when_neither_provides_context(self) -> None:
|
||||
with pytest.raises(ValueError, match="Kontext konnte nicht bestimmt werden"):
|
||||
self.router.determine_context()
|
||||
|
||||
def test_raises_when_cwd_has_no_context(self) -> None:
|
||||
cwd = Path("/some/other/directory")
|
||||
with pytest.raises(ValueError, match="Kontext konnte nicht bestimmt werden"):
|
||||
self.router.determine_context(cwd=cwd)
|
||||
|
||||
def test_raises_when_config_has_invalid_context(self) -> None:
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test",
|
||||
params={"context": "invalid"},
|
||||
)
|
||||
with pytest.raises(ValueError, match="Kontext konnte nicht bestimmt werden"):
|
||||
self.router.determine_context(source_config=config)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ContextRouter.resolve_target_path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveTargetPath:
|
||||
"""Tests for ContextRouter.resolve_target_path."""
|
||||
|
||||
def setup_method(self) -> None:
|
||||
self.router = ContextRouter(Path("/monorepo"))
|
||||
|
||||
def test_meeting_path(self) -> None:
|
||||
result = self.router.resolve_target_path("bahn", "meeting", "2024-01-15-standup.md")
|
||||
assert result == Path("/monorepo/bahn/knowledge/meetings/2024-01-15-standup.md")
|
||||
|
||||
def test_decision_path(self) -> None:
|
||||
result = self.router.resolve_target_path("dhive", "decision", "adr-001.md")
|
||||
assert result == Path("/monorepo/dhive/knowledge/decisions/adr-001.md")
|
||||
|
||||
def test_project_path(self) -> None:
|
||||
result = self.router.resolve_target_path("privat", "project", "side-project.md")
|
||||
assert result == Path("/monorepo/privat/knowledge/projects/side-project.md")
|
||||
|
||||
def test_reference_path(self) -> None:
|
||||
result = self.router.resolve_target_path("bahn", "reference", "api-docs.md")
|
||||
assert result == Path("/monorepo/bahn/knowledge/references/api-docs.md")
|
||||
|
||||
def test_link_path(self) -> None:
|
||||
result = self.router.resolve_target_path("dhive", "link", "useful-link.md")
|
||||
assert result == Path("/monorepo/dhive/knowledge/links/useful-link.md")
|
||||
|
||||
def test_unknown_category_goes_to_inbox(self) -> None:
|
||||
result = self.router.resolve_target_path("bahn", "random", "something.md")
|
||||
assert result == Path("/monorepo/bahn/knowledge/inbox/something.md")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ContextRouter.route (convenience method)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRoute:
|
||||
"""Tests for ContextRouter.route convenience method."""
|
||||
|
||||
def setup_method(self) -> None:
|
||||
self.router = ContextRouter(Path("/monorepo"))
|
||||
|
||||
def test_returns_routing_decision(self) -> None:
|
||||
config = SourceConfig(type="file", name="Test", params={"context": "bahn"})
|
||||
decision = self.router.route("meeting", "standup.md", source_config=config)
|
||||
|
||||
assert isinstance(decision, RoutingDecision)
|
||||
assert decision.context == "bahn"
|
||||
assert decision.target_folder == "meetings"
|
||||
assert decision.artifact_path == Path("/monorepo/bahn/knowledge/meetings/standup.md")
|
||||
|
||||
def test_route_with_cwd(self) -> None:
|
||||
cwd = Path("/monorepo/privat/something")
|
||||
decision = self.router.route("link", "bookmark.md", cwd=cwd)
|
||||
|
||||
assert decision.context == "privat"
|
||||
assert decision.target_folder == "links"
|
||||
assert decision.artifact_path == Path("/monorepo/privat/knowledge/links/bookmark.md")
|
||||
@@ -0,0 +1,450 @@
|
||||
"""Unit-Tests für die Jira-Quellstrategie.
|
||||
|
||||
Testet JQL-basierte Issue-Extraktion, Markdown-Komposition,
|
||||
Frontmatter-Anreicherung und inkrementelle Updates mit gemockten HTTP-Responses.
|
||||
|
||||
Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
from monorepo.knowledge.sources.jira import (
|
||||
JiraSource,
|
||||
_build_frontmatter,
|
||||
_build_issue_markdown,
|
||||
_parse_jira_date,
|
||||
_slugify,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_issue(
|
||||
key: str = "ACV2-123",
|
||||
summary: str = "Fix login bug",
|
||||
description: str = "Users cannot log in via SSO.",
|
||||
status: str = "In Progress",
|
||||
project: str = "ACV2",
|
||||
assignee: str = "Max Mustermann",
|
||||
reporter: str = "Anna Schmidt",
|
||||
updated: str = "2025-07-01T10:30:00.000+0200",
|
||||
created: str = "2025-06-15T08:00:00.000+0200",
|
||||
comments: list | None = None,
|
||||
) -> dict:
|
||||
"""Erstellt ein Jira-Issue-Dict für Tests."""
|
||||
if comments is None:
|
||||
comments = []
|
||||
return {
|
||||
"key": key,
|
||||
"fields": {
|
||||
"summary": summary,
|
||||
"description": description,
|
||||
"status": {"name": status},
|
||||
"project": {"key": project},
|
||||
"assignee": {"displayName": assignee} if assignee else None,
|
||||
"reporter": {"displayName": reporter} if reporter else None,
|
||||
"updated": updated,
|
||||
"created": created,
|
||||
"comment": {"comments": comments},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _make_comment(
|
||||
author: str = "Max Mustermann",
|
||||
body: str = "Fixed in latest commit.",
|
||||
created: str = "2025-07-01T12:00:00.000+0200",
|
||||
) -> dict:
|
||||
return {
|
||||
"author": {"displayName": author},
|
||||
"body": body,
|
||||
"created": created,
|
||||
}
|
||||
|
||||
|
||||
def _make_config(
|
||||
base_url: str = "https://jira.bahn.de",
|
||||
token: str = "test-token",
|
||||
jql: str = "project = ACV2 AND updatedDate > -7d",
|
||||
username: str = "",
|
||||
) -> SourceConfig:
|
||||
"""Erstellt eine SourceConfig für Jira-Tests."""
|
||||
params = {"base_url": base_url, "jql": jql, "api_key": token}
|
||||
if username:
|
||||
params["username"] = username
|
||||
return SourceConfig(type="jira", name="Test Jira", params=params)
|
||||
|
||||
|
||||
def _mock_response(status_code: int = 200, json_data: dict | None = None):
|
||||
"""Erstellt ein Mock-Response-Objekt."""
|
||||
mock = MagicMock()
|
||||
mock.status_code = status_code
|
||||
mock.json.return_value = json_data or {}
|
||||
mock.text = "error" if status_code >= 400 else ""
|
||||
return mock
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Hilfsfunktionen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseJiraDate:
|
||||
"""Tests für _parse_jira_date."""
|
||||
|
||||
def test_iso_datetime(self):
|
||||
result = _parse_jira_date("2025-07-01T10:30:00.000+0200")
|
||||
assert result is not None
|
||||
assert result.year == 2025
|
||||
assert result.month == 7
|
||||
assert result.day == 1
|
||||
|
||||
def test_iso_date_only(self):
|
||||
result = _parse_jira_date("2025-07-01")
|
||||
assert result is not None
|
||||
assert result.year == 2025
|
||||
|
||||
def test_none_input(self):
|
||||
assert _parse_jira_date(None) is None
|
||||
|
||||
def test_empty_string(self):
|
||||
assert _parse_jira_date("") is None
|
||||
|
||||
def test_invalid_string(self):
|
||||
assert _parse_jira_date("not-a-date") is None
|
||||
|
||||
|
||||
class TestSlugify:
|
||||
"""Tests für _slugify."""
|
||||
|
||||
def test_simple(self):
|
||||
assert _slugify("In Progress") == "in-progress"
|
||||
|
||||
def test_special_chars(self):
|
||||
assert _slugify("ACV2/Sprint-1!") == "acv2-sprint-1"
|
||||
|
||||
def test_empty(self):
|
||||
assert _slugify("") == ""
|
||||
|
||||
|
||||
class TestBuildIssueMarkdown:
|
||||
"""Tests für _build_issue_markdown."""
|
||||
|
||||
def test_basic_issue(self):
|
||||
issue = _make_issue()
|
||||
md = _build_issue_markdown(issue)
|
||||
assert "# Fix login bug" in md
|
||||
assert "**Status:** In Progress" in md
|
||||
assert "## Description" in md
|
||||
assert "Users cannot log in via SSO." in md
|
||||
|
||||
def test_issue_without_description(self):
|
||||
issue = _make_issue(description=None)
|
||||
md = _build_issue_markdown(issue)
|
||||
assert "# Fix login bug" in md
|
||||
assert "## Description" not in md
|
||||
|
||||
def test_issue_with_comments(self):
|
||||
comments = [
|
||||
_make_comment(author="Max", body="Working on it.", created="2025-07-01T12:00:00.000+0200"),
|
||||
_make_comment(author="Anna", body="Please prioritize.", created="2025-07-02T09:00:00.000+0200"),
|
||||
]
|
||||
issue = _make_issue(comments=comments)
|
||||
md = _build_issue_markdown(issue)
|
||||
assert "## Comments" in md
|
||||
assert "### Max (2025-07-01)" in md
|
||||
assert "Working on it." in md
|
||||
assert "### Anna (2025-07-02)" in md
|
||||
assert "Please prioritize." in md
|
||||
|
||||
def test_issue_without_comments(self):
|
||||
issue = _make_issue(comments=[])
|
||||
md = _build_issue_markdown(issue)
|
||||
assert "## Comments" not in md
|
||||
|
||||
|
||||
class TestBuildFrontmatter:
|
||||
"""Tests für _build_frontmatter."""
|
||||
|
||||
def test_full_issue(self):
|
||||
issue = _make_issue()
|
||||
fm = _build_frontmatter(issue, "https://jira.bahn.de")
|
||||
assert fm["type"] == "jira"
|
||||
assert fm["issue_key"] == "ACV2-123"
|
||||
assert fm["project"] == "ACV2"
|
||||
assert fm["status"] == "In Progress"
|
||||
assert fm["assignee"] == "Max Mustermann"
|
||||
assert fm["reporter"] == "Anna Schmidt"
|
||||
assert fm["url"] == "https://jira.bahn.de/browse/ACV2-123"
|
||||
|
||||
def test_no_assignee(self):
|
||||
issue = _make_issue(assignee=None)
|
||||
fm = _build_frontmatter(issue, "https://jira.bahn.de")
|
||||
assert fm["assignee"] == ""
|
||||
|
||||
def test_trailing_slash_in_url(self):
|
||||
issue = _make_issue(key="PROJ-1")
|
||||
fm = _build_frontmatter(issue, "https://jira.bahn.de/")
|
||||
assert fm["url"] == "https://jira.bahn.de/browse/PROJ-1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: JiraSource.extract()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestJiraSourceExtract:
|
||||
"""Tests für JiraSource.extract() mit gemockten HTTP-Responses."""
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_successful_extraction(self, mock_session_cls):
|
||||
"""Erfolgreiche Extraktion von Issues."""
|
||||
issues = [_make_issue(), _make_issue(key="ACV2-456", summary="Add feature")]
|
||||
response = _mock_response(200, {"issues": issues, "total": 2})
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 2
|
||||
assert result.errors == []
|
||||
assert result.skipped == 0
|
||||
|
||||
# Erstes Artefakt prüfen
|
||||
art = result.artifacts[0]
|
||||
assert art.metadata.type == "jira"
|
||||
assert art.metadata.title == "Fix login bug"
|
||||
assert art.metadata.source_context == "bahn"
|
||||
assert art.metadata.source["issue_key"] == "ACV2-123"
|
||||
assert art.metadata.source["project"] == "ACV2"
|
||||
assert art.metadata.source["status"] == "In Progress"
|
||||
assert art.metadata.source["url"] == "https://jira.bahn.de/browse/ACV2-123"
|
||||
assert art.metadata.category == "project"
|
||||
assert "acv2" in art.metadata.tags
|
||||
assert art.metadata.content_hash.startswith("sha256:")
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_incremental_skip(self, mock_session_cls):
|
||||
"""Bereits bekannte Issues werden übersprungen."""
|
||||
issue = _make_issue(updated="2025-07-01T10:30:00.000+0200")
|
||||
response = _mock_response(200, {"issues": [issue], "total": 1})
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
source.set_last_updated({"ACV2-123": "2025-07-01T10:30:00.000+0200"})
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert result.skipped == 1
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_incremental_update_when_changed(self, mock_session_cls):
|
||||
"""Geänderte Issues werden erneut verarbeitet."""
|
||||
issue = _make_issue(updated="2025-07-02T10:30:00.000+0200")
|
||||
response = _mock_response(200, {"issues": [issue], "total": 1})
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
source.set_last_updated({"ACV2-123": "2025-07-01T10:30:00.000+0200"})
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
assert result.skipped == 0
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_auth_failure(self, mock_session_cls):
|
||||
"""401-Fehler wird als Auth-Error gemeldet."""
|
||||
response = _mock_response(401)
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "auth"
|
||||
assert result.errors[0].retry is False
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_connection_error(self, mock_session_cls):
|
||||
"""Verbindungsfehler wird graceful gehandelt."""
|
||||
import requests as req
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.side_effect = req.ConnectionError("refused")
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "connection"
|
||||
assert result.errors[0].retry is True
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_server_error_retryable(self, mock_session_cls):
|
||||
"""5xx-Fehler sind retry-fähig."""
|
||||
response = _mock_response(500)
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].retry is True
|
||||
|
||||
def test_missing_base_url(self):
|
||||
"""Fehlende base_url wird als Config-Error gemeldet."""
|
||||
config = SourceConfig(
|
||||
type="jira",
|
||||
name="Bad Config",
|
||||
params={"jql": "project = X"},
|
||||
)
|
||||
source = JiraSource()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "config"
|
||||
assert "base_url" in result.errors[0].message
|
||||
|
||||
def test_missing_jql(self):
|
||||
"""Fehlende JQL wird als Config-Error gemeldet."""
|
||||
config = SourceConfig(
|
||||
type="jira",
|
||||
name="Bad Config",
|
||||
params={"base_url": "https://jira.test.de"},
|
||||
)
|
||||
source = JiraSource()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "config"
|
||||
assert "jql" in result.errors[0].message
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_pagination(self, mock_session_cls):
|
||||
"""Pagination über mehrere Seiten funktioniert."""
|
||||
page1_issues = [_make_issue(key=f"ACV2-{i}") for i in range(3)]
|
||||
page2_issues = [_make_issue(key=f"ACV2-{i}") for i in range(3, 5)]
|
||||
|
||||
resp1 = _mock_response(200, {"issues": page1_issues, "total": 5})
|
||||
resp2 = _mock_response(200, {"issues": page2_issues, "total": 5})
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.side_effect = [resp1, resp2]
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 5
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_issue_with_comments_in_artifact(self, mock_session_cls):
|
||||
"""Kommentare werden im Markdown-Body dargestellt."""
|
||||
comments = [_make_comment(author="Dev", body="Done.")]
|
||||
issue = _make_issue(comments=comments)
|
||||
response = _mock_response(200, {"issues": [issue], "total": 1})
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
assert "## Comments" in result.artifacts[0].content
|
||||
assert "Done." in result.artifacts[0].content
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_artifact_file_path_from_key(self, mock_session_cls):
|
||||
"""Dateiname wird aus Issue-Key abgeleitet."""
|
||||
issue = _make_issue(key="PROJ-99")
|
||||
response = _mock_response(200, {"issues": [issue], "total": 1})
|
||||
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert result.artifacts[0].file_path.name == "proj-99.md"
|
||||
|
||||
|
||||
class TestJiraSourceSupportsIncremental:
|
||||
"""Tests für supports_incremental()."""
|
||||
|
||||
def test_returns_true(self):
|
||||
source = JiraSource()
|
||||
assert source.supports_incremental() is True
|
||||
|
||||
|
||||
class TestJiraSourceAuth:
|
||||
"""Tests für Authentifizierungs-Konfiguration."""
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_basic_auth_with_username(self, mock_session_cls):
|
||||
"""Bei username + token wird Basic Auth verwendet."""
|
||||
response = _mock_response(200, {"issues": [], "total": 0})
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config(username="user@bahn.de", token="secret")
|
||||
source.extract(config, "bahn")
|
||||
|
||||
# Verify Basic Auth was set
|
||||
assert session_instance.auth == ("user@bahn.de", "secret")
|
||||
|
||||
@patch("monorepo.knowledge.sources.jira.requests.Session")
|
||||
def test_bearer_token_without_username(self, mock_session_cls):
|
||||
"""Ohne username wird Bearer-Token verwendet."""
|
||||
response = _mock_response(200, {"issues": [], "total": 0})
|
||||
session_instance = MagicMock()
|
||||
session_instance.get.return_value = response
|
||||
mock_session_cls.return_value = session_instance
|
||||
|
||||
source = JiraSource()
|
||||
config = _make_config(token="bearer-token")
|
||||
source.extract(config, "bahn")
|
||||
|
||||
# Verify Bearer was set in headers
|
||||
assert session_instance.headers.__setitem__.call_args_list[2] == (
|
||||
("Authorization", "Bearer bearer-token"),
|
||||
) or "Bearer" in str(session_instance.headers)
|
||||
@@ -0,0 +1,506 @@
|
||||
"""Unit-Tests: CLI-Argument-Parsing für Knowledge Management.
|
||||
|
||||
Testet alle Subcommands mit verschiedenen Flag-Kombinationen,
|
||||
Kontext-Ableitung aus CWD und Hilfe-Ausgabe.
|
||||
|
||||
Requirements: 15.2, 15.3, 15.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.cli import (
|
||||
VALID_CONTEXTS,
|
||||
_derive_context_from_cwd,
|
||||
build_knowledge_parser,
|
||||
knowledge_main,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
"""Build a standalone knowledge parser for testing."""
|
||||
return build_knowledge_parser()
|
||||
|
||||
|
||||
def _parse(parser: argparse.ArgumentParser, args: list[str]) -> argparse.Namespace:
|
||||
"""Parse args without triggering SystemExit on error."""
|
||||
return parser.parse_args(args)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: capture subcommand
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCaptureCommand:
|
||||
"""Tests for 'knowledge capture' argument parsing."""
|
||||
|
||||
def test_capture_plain_text(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with plain text input."""
|
||||
ns = _parse(parser, ["capture", "Eine schnelle Notiz"])
|
||||
assert ns.knowledge_command == "capture"
|
||||
assert ns.input == "Eine schnelle Notiz"
|
||||
assert ns.context is None
|
||||
assert ns.tags is None
|
||||
|
||||
def test_capture_url_input(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with URL input."""
|
||||
ns = _parse(parser, ["capture", "https://example.com/docs"])
|
||||
assert ns.knowledge_command == "capture"
|
||||
assert ns.input == "https://example.com/docs"
|
||||
|
||||
def test_capture_file_path_input(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with file path input."""
|
||||
ns = _parse(parser, ["capture", "/tmp/notes/meeting.md"])
|
||||
assert ns.input == "/tmp/notes/meeting.md"
|
||||
|
||||
def test_capture_with_context(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with explicit --context flag."""
|
||||
ns = _parse(parser, ["capture", "Notiz", "--context", "bahn"])
|
||||
assert ns.context == "bahn"
|
||||
|
||||
def test_capture_with_context_short(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with -c short flag."""
|
||||
ns = _parse(parser, ["capture", "Notiz", "-c", "dhive"])
|
||||
assert ns.context == "dhive"
|
||||
|
||||
def test_capture_with_tags(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with --tags flag."""
|
||||
ns = _parse(parser, ["capture", "Notiz", "--tags", "tag1,tag2"])
|
||||
assert ns.tags == "tag1,tag2"
|
||||
|
||||
def test_capture_with_tags_short(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with -t short flag."""
|
||||
ns = _parse(parser, ["capture", "Notiz", "-t", "meeting,acv2"])
|
||||
assert ns.tags == "meeting,acv2"
|
||||
|
||||
def test_capture_with_context_and_tags(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with both --context and --tags."""
|
||||
ns = _parse(parser, [
|
||||
"capture", "Sprint Retro Notiz",
|
||||
"--context", "bahn",
|
||||
"--tags", "tag1,tag2",
|
||||
])
|
||||
assert ns.context == "bahn"
|
||||
assert ns.tags == "tag1,tag2"
|
||||
assert ns.input == "Sprint Retro Notiz"
|
||||
|
||||
def test_capture_invalid_context_rejected(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Capture with invalid context raises error."""
|
||||
with pytest.raises(SystemExit):
|
||||
_parse(parser, ["capture", "Notiz", "--context", "invalid"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: ingest subcommand
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIngestCommand:
|
||||
"""Tests for 'knowledge ingest' argument parsing."""
|
||||
|
||||
def test_ingest_defaults(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Ingest with no flags uses defaults."""
|
||||
ns = _parse(parser, ["ingest"])
|
||||
assert ns.knowledge_command == "ingest"
|
||||
assert ns.context is None
|
||||
assert ns.dry_run is False
|
||||
assert ns.verbose is False
|
||||
assert ns.no_commit is False
|
||||
|
||||
def test_ingest_with_context(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Ingest with --context flag."""
|
||||
ns = _parse(parser, ["ingest", "--context", "privat"])
|
||||
assert ns.context == "privat"
|
||||
|
||||
def test_ingest_with_dry_run(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Ingest with --dry-run flag."""
|
||||
ns = _parse(parser, ["ingest", "--dry-run"])
|
||||
assert ns.dry_run is True
|
||||
|
||||
def test_ingest_with_verbose(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Ingest with --verbose flag."""
|
||||
ns = _parse(parser, ["ingest", "--verbose"])
|
||||
assert ns.verbose is True
|
||||
|
||||
def test_ingest_with_verbose_short(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Ingest with -v short flag."""
|
||||
ns = _parse(parser, ["ingest", "-v"])
|
||||
assert ns.verbose is True
|
||||
|
||||
def test_ingest_with_no_commit(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Ingest with --no-commit flag."""
|
||||
ns = _parse(parser, ["ingest", "--no-commit"])
|
||||
assert ns.no_commit is True
|
||||
|
||||
def test_ingest_all_flags(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Ingest with all flags combined."""
|
||||
ns = _parse(parser, [
|
||||
"ingest",
|
||||
"--context", "privat",
|
||||
"--dry-run",
|
||||
"--verbose",
|
||||
"--no-commit",
|
||||
])
|
||||
assert ns.context == "privat"
|
||||
assert ns.dry_run is True
|
||||
assert ns.verbose is True
|
||||
assert ns.no_commit is True
|
||||
|
||||
def test_ingest_invalid_context_rejected(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Ingest with invalid context raises error."""
|
||||
with pytest.raises(SystemExit):
|
||||
_parse(parser, ["ingest", "--context", "ungueltig"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: search subcommand
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSearchCommand:
|
||||
"""Tests for 'knowledge search' argument parsing."""
|
||||
|
||||
def test_search_basic(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Search with query string."""
|
||||
ns = _parse(parser, ["search", "API Design"])
|
||||
assert ns.knowledge_command == "search"
|
||||
assert ns.query == "API Design"
|
||||
assert ns.context is None
|
||||
assert ns.limit is None
|
||||
|
||||
def test_search_with_context(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Search with --context flag."""
|
||||
ns = _parse(parser, ["search", "query", "--context", "dhive"])
|
||||
assert ns.context == "dhive"
|
||||
|
||||
def test_search_with_limit(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Search with --limit flag."""
|
||||
ns = _parse(parser, ["search", "query", "--limit", "5"])
|
||||
assert ns.limit == 5
|
||||
|
||||
def test_search_with_limit_short(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Search with -l short flag."""
|
||||
ns = _parse(parser, ["search", "query", "-l", "10"])
|
||||
assert ns.limit == 10
|
||||
|
||||
def test_search_all_flags(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Search with --context and --limit combined."""
|
||||
ns = _parse(parser, [
|
||||
"search", "Confluence Migration",
|
||||
"--context", "bahn",
|
||||
"--limit", "5",
|
||||
])
|
||||
assert ns.query == "Confluence Migration"
|
||||
assert ns.context == "bahn"
|
||||
assert ns.limit == 5
|
||||
|
||||
def test_search_missing_query_rejected(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Search without query raises error."""
|
||||
with pytest.raises(SystemExit):
|
||||
_parse(parser, ["search"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: status subcommand
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStatusCommand:
|
||||
"""Tests for 'knowledge status' argument parsing."""
|
||||
|
||||
def test_status_defaults(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Status with no flags."""
|
||||
ns = _parse(parser, ["status"])
|
||||
assert ns.knowledge_command == "status"
|
||||
assert ns.context is None
|
||||
|
||||
def test_status_with_context(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Status with --context flag."""
|
||||
ns = _parse(parser, ["status", "--context", "bahn"])
|
||||
assert ns.context == "bahn"
|
||||
|
||||
def test_status_with_context_short(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Status with -c short flag."""
|
||||
ns = _parse(parser, ["status", "-c", "privat"])
|
||||
assert ns.context == "privat"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: link subcommand
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLinkCommand:
|
||||
"""Tests for 'knowledge link' argument parsing."""
|
||||
|
||||
def test_link_basic(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Link with URL only."""
|
||||
ns = _parse(parser, ["link", "https://example.com"])
|
||||
assert ns.knowledge_command == "link"
|
||||
assert ns.url == "https://example.com"
|
||||
assert ns.title is None
|
||||
assert ns.tags is None
|
||||
assert ns.context is None
|
||||
|
||||
def test_link_with_title(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Link with --title flag."""
|
||||
ns = _parse(parser, ["link", "https://example.com", "--title", "Example Page"])
|
||||
assert ns.title == "Example Page"
|
||||
|
||||
def test_link_with_title_short(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Link with -T short flag."""
|
||||
ns = _parse(parser, ["link", "https://example.com", "-T", "Example"])
|
||||
assert ns.title == "Example"
|
||||
|
||||
def test_link_with_tags(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Link with --tags flag."""
|
||||
ns = _parse(parser, ["link", "https://example.com", "--tags", "ref"])
|
||||
assert ns.tags == "ref"
|
||||
|
||||
def test_link_with_context(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Link with --context flag."""
|
||||
ns = _parse(parser, ["link", "https://example.com", "--context", "bahn"])
|
||||
assert ns.context == "bahn"
|
||||
|
||||
def test_link_all_flags(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Link with all flags combined."""
|
||||
ns = _parse(parser, [
|
||||
"link", "https://example.com/docs",
|
||||
"--title", "Documentation",
|
||||
"--tags", "ref,docs",
|
||||
"--context", "dhive",
|
||||
])
|
||||
assert ns.url == "https://example.com/docs"
|
||||
assert ns.title == "Documentation"
|
||||
assert ns.tags == "ref,docs"
|
||||
assert ns.context == "dhive"
|
||||
|
||||
def test_link_missing_url_rejected(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Link without URL raises error."""
|
||||
with pytest.raises(SystemExit):
|
||||
_parse(parser, ["link"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Context derivation from CWD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestContextDerivation:
|
||||
"""Tests for _derive_context_from_cwd."""
|
||||
|
||||
def test_derive_bahn_context(self) -> None:
|
||||
"""CWD in bahn/ subtree returns 'bahn'."""
|
||||
with patch("monorepo.knowledge.cli.Path") as mock_path:
|
||||
mock_path.cwd.return_value = Path("/repo/bahn/aisupport/src")
|
||||
result = _derive_context_from_cwd()
|
||||
assert result == "bahn"
|
||||
|
||||
def test_derive_dhive_context(self) -> None:
|
||||
"""CWD in dhive/ subtree returns 'dhive'."""
|
||||
with patch("monorepo.knowledge.cli.Path") as mock_path:
|
||||
mock_path.cwd.return_value = Path("/repo/dhive/project/src")
|
||||
result = _derive_context_from_cwd()
|
||||
assert result == "dhive"
|
||||
|
||||
def test_derive_privat_context(self) -> None:
|
||||
"""CWD in privat/ subtree returns 'privat'."""
|
||||
with patch("monorepo.knowledge.cli.Path") as mock_path:
|
||||
mock_path.cwd.return_value = Path("/repo/privat/notes")
|
||||
result = _derive_context_from_cwd()
|
||||
assert result == "privat"
|
||||
|
||||
def test_derive_no_context(self) -> None:
|
||||
"""CWD outside context folders returns None."""
|
||||
with patch("monorepo.knowledge.cli.Path") as mock_path:
|
||||
mock_path.cwd.return_value = Path("/repo/shared/tools")
|
||||
result = _derive_context_from_cwd()
|
||||
assert result is None
|
||||
|
||||
def test_derive_bahn_at_end(self) -> None:
|
||||
"""CWD ending with /bahn returns 'bahn'."""
|
||||
with patch("monorepo.knowledge.cli.Path") as mock_path:
|
||||
mock_path.cwd.return_value = Path("/repo/bahn")
|
||||
result = _derive_context_from_cwd()
|
||||
assert result == "bahn"
|
||||
|
||||
def test_derive_windows_path(self) -> None:
|
||||
"""CWD as Windows path with backslashes works."""
|
||||
with patch("monorepo.knowledge.cli.Path") as mock_path:
|
||||
mock_path.cwd.return_value = Path("C:\\Users\\user\\repo\\bahn\\project")
|
||||
result = _derive_context_from_cwd()
|
||||
assert result == "bahn"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Help output (no arguments)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHelpOutput:
|
||||
"""Tests for help behavior when called without arguments."""
|
||||
|
||||
def test_no_command_sets_none(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""No subcommand results in knowledge_command=None."""
|
||||
ns = _parse(parser, [])
|
||||
assert ns.knowledge_command is None
|
||||
|
||||
def test_knowledge_main_no_command_shows_help(self) -> None:
|
||||
"""knowledge_main with no command triggers help (SystemExit)."""
|
||||
parser = build_knowledge_parser()
|
||||
ns = parser.parse_args([])
|
||||
# knowledge_main should call sys.exit when no command given
|
||||
with pytest.raises(SystemExit):
|
||||
knowledge_main(ns)
|
||||
|
||||
def test_parser_has_all_subcommands(self, parser: argparse.ArgumentParser) -> None:
|
||||
"""Parser registers all expected subcommands."""
|
||||
# Verify by parsing each command without error
|
||||
commands = {
|
||||
"capture": ["capture", "test input"],
|
||||
"ingest": ["ingest"],
|
||||
"search": ["search", "query"],
|
||||
"status": ["status"],
|
||||
"link": ["link", "https://example.com"],
|
||||
}
|
||||
for cmd_name, args in commands.items():
|
||||
ns = _parse(parser, args)
|
||||
assert ns.knowledge_command == cmd_name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Handler dispatch with mocks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHandlerDispatch:
|
||||
"""Tests that handlers are correctly dispatched and call underlying components."""
|
||||
|
||||
def test_capture_handler_calls_quick_capture(self, tmp_path: Path) -> None:
|
||||
"""capture handler invokes QuickCapture.capture."""
|
||||
parser = build_knowledge_parser()
|
||||
ns = parser.parse_args(["capture", "Test note", "--context", "bahn"])
|
||||
|
||||
with patch("monorepo.knowledge.cli._find_monorepo_root", return_value=tmp_path), \
|
||||
patch("monorepo.knowledge.ingestion.capture.QuickCapture") as mock_qc_cls:
|
||||
mock_qc = MagicMock()
|
||||
mock_qc.capture.return_value = tmp_path / "bahn/knowledge/inbox/note.md"
|
||||
mock_qc_cls.return_value = mock_qc
|
||||
|
||||
from monorepo.knowledge.cli import _cmd_capture
|
||||
result = _cmd_capture(ns)
|
||||
|
||||
assert result == 0
|
||||
mock_qc.capture.assert_called_once_with(
|
||||
input_text="Test note",
|
||||
context="bahn",
|
||||
tags=[],
|
||||
)
|
||||
|
||||
def test_capture_handler_splits_tags(self, tmp_path: Path) -> None:
|
||||
"""capture handler correctly splits comma-separated tags."""
|
||||
parser = build_knowledge_parser()
|
||||
ns = parser.parse_args(["capture", "Notiz", "-c", "dhive", "-t", "a,b,c"])
|
||||
|
||||
with patch("monorepo.knowledge.cli._find_monorepo_root", return_value=tmp_path), \
|
||||
patch("monorepo.knowledge.ingestion.capture.QuickCapture") as mock_qc_cls:
|
||||
mock_qc = MagicMock()
|
||||
mock_qc.capture.return_value = tmp_path / "dhive/knowledge/inbox/note.md"
|
||||
mock_qc_cls.return_value = mock_qc
|
||||
|
||||
from monorepo.knowledge.cli import _cmd_capture
|
||||
_cmd_capture(ns)
|
||||
|
||||
mock_qc.capture.assert_called_once_with(
|
||||
input_text="Notiz",
|
||||
context="dhive",
|
||||
tags=["a", "b", "c"],
|
||||
)
|
||||
|
||||
def test_ingest_handler_calls_pipeline(self, tmp_path: Path) -> None:
|
||||
"""ingest handler invokes IngestionPipeline.run."""
|
||||
parser = build_knowledge_parser()
|
||||
ns = parser.parse_args(["ingest", "--context", "privat", "--dry-run", "--no-commit"])
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.context = "privat"
|
||||
mock_result.processed = 5
|
||||
mock_result.updated = 0
|
||||
mock_result.skipped = 2
|
||||
mock_result.errors = []
|
||||
|
||||
with patch("monorepo.knowledge.cli._find_monorepo_root", return_value=tmp_path), \
|
||||
patch("monorepo.knowledge.ingestion.pipeline.IngestionPipeline") as mock_pipeline_cls:
|
||||
mock_pipeline = MagicMock()
|
||||
mock_pipeline.run.return_value = mock_result
|
||||
mock_pipeline_cls.return_value = mock_pipeline
|
||||
|
||||
from monorepo.knowledge.cli import _cmd_ingest
|
||||
result = _cmd_ingest(ns)
|
||||
|
||||
assert result == 0
|
||||
mock_pipeline_cls.assert_called_once_with(
|
||||
monorepo_root=tmp_path,
|
||||
context="privat",
|
||||
dry_run=True,
|
||||
no_commit=True,
|
||||
)
|
||||
mock_pipeline.run.assert_called_once()
|
||||
|
||||
def test_status_handler_counts_inbox(self, tmp_path: Path) -> None:
|
||||
"""status handler counts .md files in inbox (excluding index.md)."""
|
||||
# Create inbox with some files
|
||||
inbox = tmp_path / "bahn" / "knowledge" / "inbox"
|
||||
inbox.mkdir(parents=True)
|
||||
(inbox / "index.md").write_text("# Inbox")
|
||||
(inbox / "note1.md").write_text("note 1")
|
||||
(inbox / "note2.md").write_text("note 2")
|
||||
|
||||
parser = build_knowledge_parser()
|
||||
ns = parser.parse_args(["status", "--context", "bahn"])
|
||||
|
||||
with patch("monorepo.knowledge.cli._find_monorepo_root", return_value=tmp_path):
|
||||
from monorepo.knowledge.cli import _cmd_status
|
||||
result = _cmd_status(ns)
|
||||
|
||||
assert result == 0
|
||||
|
||||
def test_link_handler_calls_registry(self, tmp_path: Path) -> None:
|
||||
"""link handler invokes LinkRegistry.save_link."""
|
||||
parser = build_knowledge_parser()
|
||||
ns = parser.parse_args([
|
||||
"link", "https://example.com",
|
||||
"--title", "Example",
|
||||
"--tags", "ref",
|
||||
"--context", "bahn",
|
||||
])
|
||||
|
||||
with patch("monorepo.knowledge.cli._find_monorepo_root", return_value=tmp_path), \
|
||||
patch("monorepo.knowledge.sources.link.LinkRegistry") as mock_reg_cls:
|
||||
mock_reg = MagicMock()
|
||||
mock_reg.save_link.return_value = tmp_path / "bahn/knowledge/links/example.md"
|
||||
mock_reg_cls.return_value = mock_reg
|
||||
|
||||
from monorepo.knowledge.cli import _cmd_link
|
||||
result = _cmd_link(ns)
|
||||
|
||||
assert result == 0
|
||||
mock_reg_cls.assert_called_once_with(tmp_path / "bahn" / "knowledge")
|
||||
mock_reg.save_link.assert_called_once_with(
|
||||
url="https://example.com",
|
||||
title="Example",
|
||||
tags=["ref"],
|
||||
)
|
||||
@@ -0,0 +1,488 @@
|
||||
"""Tests für kontextübergreifende Suche und Agent-Context-Injection.
|
||||
|
||||
Validiert:
|
||||
- search_cross_context: Suche über alle Knowledge-Folder _index.yaml-Dateien
|
||||
- Scope-Filterung: Nur autorisierte Kontexte in Ergebnissen
|
||||
- get_context_injection: Max. 10 relevante Artefakt-Pfade, sortiert nach Relevanz
|
||||
- Progressive Disclosure: Schicht 1 (Index) für Relevanz, Schicht 2 (Datei) bei Bedarf
|
||||
- load_artifact_content: Schicht-2-Nachladen
|
||||
|
||||
Requirements: 11.1, 11.2, 11.3, 11.5, 11.6
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.store import (
|
||||
ContextInjectionResult,
|
||||
KnowledgeStore,
|
||||
MAX_CONTEXT_INJECTION_RESULTS,
|
||||
SearchResult,
|
||||
)
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scope_config() -> ScopeConfig:
|
||||
"""Beispiel-ScopeConfig."""
|
||||
return ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def monorepo_root(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine simulierte Monorepo-Struktur mit Knowledge-Folder-Indices."""
|
||||
# bahn/knowledge/_index.yaml
|
||||
bahn_knowledge = tmp_path / "bahn" / "knowledge"
|
||||
bahn_knowledge.mkdir(parents=True)
|
||||
bahn_index_data = {
|
||||
"version": "1.0",
|
||||
"last_updated": "2025-07-01T10:00:00Z",
|
||||
"artifacts": [
|
||||
{
|
||||
"id": "bahn/meetings/sprint-planning-w24",
|
||||
"title": "Sprint Planning ACV2 W24",
|
||||
"type": "meeting",
|
||||
"tags": ["acv2", "sprint", "planning"],
|
||||
"scope": "bahn",
|
||||
"summary": "Sprint Planning für ACV2 Migration, Woche 24",
|
||||
"path": "bahn/knowledge/meetings/sprint-planning-w24.md",
|
||||
"content_hash": "sha256:aaa111",
|
||||
"links": [],
|
||||
},
|
||||
{
|
||||
"id": "bahn/decisions/api-redesign",
|
||||
"title": "API Redesign Entscheidung",
|
||||
"type": "decision",
|
||||
"tags": ["api", "architecture", "rest"],
|
||||
"scope": "bahn",
|
||||
"summary": "REST-API-Konventionen für neue Microservices",
|
||||
"path": "bahn/knowledge/decisions/api-redesign.md",
|
||||
"content_hash": "sha256:bbb222",
|
||||
"links": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
with open(bahn_knowledge / "_index.yaml", "w", encoding="utf-8") as f:
|
||||
yaml.dump(bahn_index_data, f, allow_unicode=True)
|
||||
|
||||
# dhive/knowledge/_index.yaml
|
||||
dhive_knowledge = tmp_path / "dhive" / "knowledge"
|
||||
dhive_knowledge.mkdir(parents=True)
|
||||
dhive_index_data = {
|
||||
"version": "1.0",
|
||||
"last_updated": "2025-07-01T11:00:00Z",
|
||||
"artifacts": [
|
||||
{
|
||||
"id": "dhive/projects/jury-voting-backend",
|
||||
"title": "Jury Voting Backend",
|
||||
"type": "project",
|
||||
"tags": ["jury-voting", "backend", "api"],
|
||||
"scope": "dhive",
|
||||
"summary": "Backend-Architektur für das Jury-Voting-System",
|
||||
"path": "dhive/knowledge/projects/jury-voting-backend.md",
|
||||
"content_hash": "sha256:ccc333",
|
||||
"links": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
with open(dhive_knowledge / "_index.yaml", "w", encoding="utf-8") as f:
|
||||
yaml.dump(dhive_index_data, f, allow_unicode=True)
|
||||
|
||||
# privat/knowledge/_index.yaml
|
||||
privat_knowledge = tmp_path / "privat" / "knowledge"
|
||||
privat_knowledge.mkdir(parents=True)
|
||||
privat_index_data = {
|
||||
"version": "1.0",
|
||||
"last_updated": "2025-07-01T12:00:00Z",
|
||||
"artifacts": [
|
||||
{
|
||||
"id": "privat/references/rest-patterns",
|
||||
"title": "REST Patterns Sammlung",
|
||||
"type": "reference",
|
||||
"tags": ["api", "rest", "patterns"],
|
||||
"scope": "privat",
|
||||
"summary": "Sammlung bewährter REST-API-Patterns",
|
||||
"path": "privat/knowledge/references/rest-patterns.md",
|
||||
"content_hash": "sha256:ddd444",
|
||||
"links": [],
|
||||
},
|
||||
{
|
||||
"id": "privat/notes/geheime-notiz",
|
||||
"title": "Private Notiz",
|
||||
"type": "note",
|
||||
"tags": ["persönlich", "ideen"],
|
||||
"scope": "privat",
|
||||
"summary": "Persönliche Gedanken und Ideen",
|
||||
"path": "privat/knowledge/notes/geheime-notiz.md",
|
||||
"content_hash": "sha256:eee555",
|
||||
"links": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
with open(privat_knowledge / "_index.yaml", "w", encoding="utf-8") as f:
|
||||
yaml.dump(privat_index_data, f, allow_unicode=True)
|
||||
|
||||
# Erstelle auch eine Artefakt-Datei für Schicht-2-Tests
|
||||
meetings_dir = bahn_knowledge / "meetings"
|
||||
meetings_dir.mkdir(parents=True, exist_ok=True)
|
||||
(meetings_dir / "sprint-planning-w24.md").write_text(
|
||||
"---\ntype: meeting\ntitle: Sprint Planning ACV2 W24\n---\n\n# Sprint Planning\n\nInhalt...\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return tmp_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Kontextübergreifende Suche
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSearchCrossContext:
|
||||
"""Tests für search_cross_context()."""
|
||||
|
||||
def test_searches_all_knowledge_folders(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() durchsucht alle Knowledge-Folder-Indices."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# "api" kommt in bahn, dhive und privat vor
|
||||
results = store.search_cross_context("api", ["bahn", "dhive", "privat"])
|
||||
|
||||
# Mindestens Einträge aus mehreren Kontexten
|
||||
scopes_found = {r.entry.scope for r in results}
|
||||
assert "bahn" in scopes_found
|
||||
assert "dhive" in scopes_found
|
||||
assert "privat" in scopes_found
|
||||
|
||||
def test_respects_scope_filter(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() liefert nur Ergebnisse aus autorisierten Scopes."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Nur bahn-Scope zugelassen
|
||||
results = store.search_cross_context("api", ["bahn"])
|
||||
|
||||
for r in results:
|
||||
assert r.entry.scope == "bahn"
|
||||
|
||||
def test_does_not_reveal_unauthorized_scopes(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() gibt keine Hinweise auf nicht-autorisierte Scopes."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Suche nach "geheime" – existiert nur in privat
|
||||
results = store.search_cross_context("geheime", ["bahn", "dhive"])
|
||||
|
||||
# Kein privat-Ergebnis trotz Existenz
|
||||
assert len(results) == 0
|
||||
|
||||
def test_returns_sorted_by_relevance(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() sortiert nach Relevanz-Score absteigend."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
results = store.search_cross_context("api", ["bahn", "dhive", "privat"])
|
||||
|
||||
# Prüfe absteigend sortiert
|
||||
for i in range(len(results) - 1):
|
||||
assert results[i].relevance_score >= results[i + 1].relevance_score
|
||||
|
||||
def test_empty_query_returns_empty(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() mit leerem Query gibt leere Liste zurück."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
results = store.search_cross_context("", ["bahn", "dhive", "privat"])
|
||||
assert results == []
|
||||
|
||||
def test_deduplication_across_indices(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() dedupliziert Einträge die im eigenen Index und Folder-Index sind."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Füge einen Eintrag zum eigenen Index hinzu, der auch im bahn-Folder existiert
|
||||
entry = IndexEntry(
|
||||
id="bahn/meetings/sprint-planning-w24",
|
||||
title="Sprint Planning ACV2 W24",
|
||||
type="meeting",
|
||||
tags=["acv2", "sprint"],
|
||||
scope="bahn",
|
||||
summary="Sprint Planning",
|
||||
path="bahn/knowledge/meetings/sprint-planning-w24.md",
|
||||
content_hash="sha256:aaa111",
|
||||
)
|
||||
store.index.update_entry(entry)
|
||||
|
||||
results = store.search_cross_context("sprint", ["bahn"])
|
||||
|
||||
# Nur ein Ergebnis, nicht zwei
|
||||
ids = [r.entry.id for r in results]
|
||||
assert ids.count("bahn/meetings/sprint-planning-w24") == 1
|
||||
|
||||
def test_handles_missing_knowledge_folder(
|
||||
self, tmp_path: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() funktioniert auch ohne existierende Knowledge-Folder."""
|
||||
# Nur den base_path erstellen, keine Knowledge-Folder
|
||||
store = KnowledgeStore(tmp_path, scope_config)
|
||||
|
||||
# Soll nicht fehlschlagen
|
||||
results = store.search_cross_context("test", ["bahn", "dhive", "privat"])
|
||||
assert results == []
|
||||
|
||||
def test_timeout_marks_partial_results(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() markiert bei Timeout alle Ergebnisse als partial."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Timeout 0 → sofort abbrechen
|
||||
results = store.search_cross_context("api", ["bahn", "dhive", "privat"], timeout=0.0)
|
||||
|
||||
# Alle gefundenen Ergebnisse sind partial (oder leer bei sofortigem Timeout)
|
||||
for r in results:
|
||||
assert r.partial_results is True
|
||||
|
||||
def test_includes_own_index_entries(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""search_cross_context() durchsucht auch den eigenen Index."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Eintrag nur im eigenen Index
|
||||
store.index.update_entry(IndexEntry(
|
||||
id="shared/docs/cli-guide",
|
||||
title="CLI Guide Dokumentation",
|
||||
type="reference",
|
||||
tags=["cli", "docs"],
|
||||
scope="shared",
|
||||
summary="Anleitung für das Monorepo-CLI",
|
||||
path="shared/docs/cli-guide.md",
|
||||
content_hash="sha256:fff666",
|
||||
))
|
||||
|
||||
results = store.search_cross_context("cli", ["shared", "bahn"])
|
||||
|
||||
ids = [r.entry.id for r in results]
|
||||
assert "shared/docs/cli-guide" in ids
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Agent-Context-Injection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetContextInjection:
|
||||
"""Tests für get_context_injection()."""
|
||||
|
||||
def test_returns_context_injection_result(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""get_context_injection() gibt ein ContextInjectionResult zurück."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
result = store.get_context_injection("api", ["bahn", "dhive", "privat"])
|
||||
|
||||
assert isinstance(result, ContextInjectionResult)
|
||||
assert isinstance(result.paths, list)
|
||||
assert isinstance(result.entries, list)
|
||||
|
||||
def test_max_10_results(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""get_context_injection() liefert max. 10 Pfade."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Viele Einträge hinzufügen
|
||||
bahn_knowledge = monorepo_root / "bahn" / "knowledge"
|
||||
entries = []
|
||||
for i in range(20):
|
||||
entries.append({
|
||||
"id": f"bahn/references/item-{i}",
|
||||
"title": f"API Reference Item {i}",
|
||||
"type": "reference",
|
||||
"tags": ["api", "reference"],
|
||||
"scope": "bahn",
|
||||
"summary": f"API-Dokumentation Teil {i}",
|
||||
"path": f"bahn/knowledge/references/item-{i}.md",
|
||||
"content_hash": f"sha256:hash{i}",
|
||||
"links": [],
|
||||
})
|
||||
|
||||
index_data = {
|
||||
"version": "1.0",
|
||||
"last_updated": "2025-07-01T14:00:00Z",
|
||||
"artifacts": entries,
|
||||
}
|
||||
with open(bahn_knowledge / "_index.yaml", "w", encoding="utf-8") as f:
|
||||
yaml.dump(index_data, f, allow_unicode=True)
|
||||
|
||||
result = store.get_context_injection("api", ["bahn"])
|
||||
|
||||
assert len(result.paths) <= MAX_CONTEXT_INJECTION_RESULTS
|
||||
assert len(result.paths) <= 10
|
||||
|
||||
def test_sorted_by_relevance(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""get_context_injection() sortiert nach Relevanz-Score absteigend."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
result = store.get_context_injection("api", ["bahn", "dhive", "privat"])
|
||||
|
||||
# Entries sind nach Relevanz sortiert
|
||||
for i in range(len(result.entries) - 1):
|
||||
assert result.entries[i].relevance_score >= result.entries[i + 1].relevance_score
|
||||
|
||||
def test_only_paths_with_valid_path(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""get_context_injection() gibt nur Pfade für Einträge mit gültigem path zurück."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Eintrag ohne path hinzufügen
|
||||
store.index.update_entry(IndexEntry(
|
||||
id="bahn/orphan",
|
||||
title="Orphan API Entry",
|
||||
type="note",
|
||||
tags=["api"],
|
||||
scope="bahn",
|
||||
path="", # Kein Pfad
|
||||
))
|
||||
|
||||
result = store.get_context_injection("api", ["bahn"])
|
||||
|
||||
# Orphan-Eintrag darf nicht in paths sein
|
||||
assert "" not in result.paths
|
||||
|
||||
def test_respects_scope_filter(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""get_context_injection() respektiert Scope-Filterung."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Nur dhive-Scope
|
||||
result = store.get_context_injection("api", ["dhive"])
|
||||
|
||||
for entry_result in result.entries:
|
||||
assert entry_result.entry.scope == "dhive"
|
||||
|
||||
def test_custom_max_results(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""get_context_injection() respektiert custom max_results."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
result = store.get_context_injection("api", ["bahn", "dhive", "privat"], max_results=2)
|
||||
|
||||
assert len(result.paths) <= 2
|
||||
|
||||
def test_empty_query_returns_empty(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""get_context_injection() mit leerem Query gibt leeres Ergebnis."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
result = store.get_context_injection("", ["bahn", "dhive", "privat"])
|
||||
|
||||
assert result.paths == []
|
||||
assert result.entries == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Progressive Disclosure (Schicht 2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLoadArtifactContent:
|
||||
"""Tests für load_artifact_content() (Progressive Disclosure Schicht 2)."""
|
||||
|
||||
def test_loads_existing_file(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""load_artifact_content() lädt den vollständigen Dateiinhalt."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
content = store.load_artifact_content("bahn/knowledge/meetings/sprint-planning-w24.md")
|
||||
|
||||
assert content is not None
|
||||
assert "Sprint Planning" in content
|
||||
assert "type: meeting" in content
|
||||
|
||||
def test_returns_none_for_missing_file(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""load_artifact_content() gibt None zurück wenn Datei nicht existiert."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
content = store.load_artifact_content("nonexistent/path/file.md")
|
||||
|
||||
assert content is None
|
||||
|
||||
def test_returns_none_for_empty_path(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""load_artifact_content() gibt None für leeren Pfad zurück."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
content = store.load_artifact_content("")
|
||||
|
||||
assert content is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Integration – Suche + Context Injection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIntegrationSearchAndInjection:
|
||||
"""Integration-Tests für das Zusammenspiel von Suche und Injection."""
|
||||
|
||||
def test_full_workflow_search_to_content(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""Vollständiger Workflow: Suche → Pfade → Inhalte nachladen."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Schicht 1: Context Injection liefert Pfade
|
||||
injection = store.get_context_injection("sprint", ["bahn"])
|
||||
|
||||
assert len(injection.paths) >= 1
|
||||
assert "bahn/knowledge/meetings/sprint-planning-w24.md" in injection.paths
|
||||
|
||||
# Schicht 2: Inhalt nachladen
|
||||
content = store.load_artifact_content(injection.paths[0])
|
||||
assert content is not None
|
||||
assert "Sprint Planning" in content
|
||||
|
||||
def test_scope_isolation_end_to_end(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""End-to-End: Scope-Isolation verhindert Zugriff auf nicht-autorisierte Artefakte."""
|
||||
store = KnowledgeStore(monorepo_root, scope_config)
|
||||
|
||||
# Suche nach privat-Artefakt mit nur bahn-Berechtigung
|
||||
injection = store.get_context_injection("geheime", ["bahn"])
|
||||
|
||||
assert len(injection.paths) == 0
|
||||
assert len(injection.entries) == 0
|
||||
@@ -0,0 +1,366 @@
|
||||
"""Tests for LinkRegistry – Web-Link-Management als Wissensartefakte.
|
||||
|
||||
Requirements: 8.1, 8.2, 8.4, 8.5, 8.6, 8.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.sources.link import LinkRegistry
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def context_path(tmp_path: Path) -> Path:
|
||||
"""Create a minimal knowledge context path."""
|
||||
knowledge = tmp_path / "bahn" / "knowledge"
|
||||
knowledge.mkdir(parents=True)
|
||||
return knowledge
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry(context_path: Path) -> LinkRegistry:
|
||||
"""Create a LinkRegistry instance."""
|
||||
return LinkRegistry(context_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# save_link tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSaveLink:
|
||||
"""Tests for save_link."""
|
||||
|
||||
def test_save_link_creates_file(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Req 8.1: Links stored as artifacts of type link in links/ folder."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("Example Page", "A description")):
|
||||
result = registry.save_link("https://example.com", title="Example")
|
||||
|
||||
assert result.exists()
|
||||
assert result.parent == context_path / "links"
|
||||
assert result.suffix == ".md"
|
||||
|
||||
def test_save_link_frontmatter_contains_required_fields(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Req 8.2: Frontmatter must contain url, title, tags, created, type: link."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("", "")):
|
||||
result = registry.save_link(
|
||||
"https://example.com/page",
|
||||
title="My Link",
|
||||
tags=["python", "docs"],
|
||||
)
|
||||
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "type: link" in content
|
||||
assert "title: My Link" in content
|
||||
assert "url: https://example.com/page" in content
|
||||
assert "python" in content
|
||||
assert "docs" in content
|
||||
assert "created:" in content
|
||||
|
||||
def test_save_link_with_tags(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Tags are included in the frontmatter."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("", "")):
|
||||
result = registry.save_link(
|
||||
"https://example.com",
|
||||
title="Test",
|
||||
tags=["tag1", "tag2"],
|
||||
)
|
||||
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "tag1" in content
|
||||
assert "tag2" in content
|
||||
|
||||
def test_save_link_empty_tags(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Links can be saved without tags."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("", "")):
|
||||
result = registry.save_link("https://example.com", title="No Tags")
|
||||
|
||||
assert result.exists()
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "tags: []" in content
|
||||
|
||||
def test_save_link_uses_fetched_title_if_none(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Req 8.4: Fetch title from URL when not provided."""
|
||||
with patch.object(
|
||||
registry, "_fetch_metadata", return_value=("Fetched Title", "Desc")
|
||||
):
|
||||
result = registry.save_link("https://example.com")
|
||||
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "Fetched Title" in content
|
||||
|
||||
def test_save_link_fetch_failed_sets_flag(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Req 8.5: Set fetch_failed: true if URL is unreachable."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("", "")):
|
||||
result = registry.save_link("https://unreachable.example.com")
|
||||
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "fetch_failed: true" in content
|
||||
|
||||
def test_save_link_creates_links_directory(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""links/ directory is created if it doesn't exist."""
|
||||
assert not (context_path / "links").exists()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("", "")):
|
||||
result = registry.save_link("https://example.com", title="Test")
|
||||
|
||||
assert (context_path / "links").exists()
|
||||
assert result.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deduplication tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDeduplication:
|
||||
"""Tests for URL deduplication with tag merging. Req 8.7."""
|
||||
|
||||
def test_duplicate_url_merges_tags(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Req 8.7: Duplicate URLs merge tags instead of creating new artifact."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("Page", "")):
|
||||
first = registry.save_link(
|
||||
"https://example.com/dup",
|
||||
title="Page",
|
||||
tags=["tag1", "tag2"],
|
||||
)
|
||||
second = registry.save_link(
|
||||
"https://example.com/dup",
|
||||
title="Page Updated",
|
||||
tags=["tag2", "tag3"],
|
||||
)
|
||||
|
||||
# Should be the same file
|
||||
assert first == second
|
||||
|
||||
content = first.read_text(encoding="utf-8")
|
||||
assert "tag1" in content
|
||||
assert "tag2" in content
|
||||
assert "tag3" in content
|
||||
|
||||
def test_duplicate_url_preserves_existing_tags(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Existing tags are preserved when merging."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("Page", "")):
|
||||
registry.save_link(
|
||||
"https://example.com/keep",
|
||||
title="Keep",
|
||||
tags=["original"],
|
||||
)
|
||||
result = registry.save_link(
|
||||
"https://example.com/keep",
|
||||
title="Keep",
|
||||
tags=["new"],
|
||||
)
|
||||
|
||||
content = result.read_text(encoding="utf-8")
|
||||
assert "original" in content
|
||||
assert "new" in content
|
||||
|
||||
def test_duplicate_url_no_change_if_tags_same(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""No rewrite if tags are already the same."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("Page", "")):
|
||||
first = registry.save_link(
|
||||
"https://example.com/same",
|
||||
title="Same",
|
||||
tags=["a", "b"],
|
||||
)
|
||||
content_before = first.read_text(encoding="utf-8")
|
||||
|
||||
second = registry.save_link(
|
||||
"https://example.com/same",
|
||||
title="Same",
|
||||
tags=["a", "b"],
|
||||
)
|
||||
content_after = second.read_text(encoding="utf-8")
|
||||
|
||||
assert first == second
|
||||
assert content_before == content_after
|
||||
|
||||
def test_different_urls_create_separate_files(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Different URLs create separate link artifacts."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("P", "")):
|
||||
first = registry.save_link("https://a.com", title="A")
|
||||
second = registry.save_link("https://b.com", title="B")
|
||||
|
||||
assert first != second
|
||||
assert first.exists()
|
||||
assert second.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# search_links tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSearchLinks:
|
||||
"""Tests for search_links. Req 8.6."""
|
||||
|
||||
def _create_link(
|
||||
self, registry: LinkRegistry, url: str, title: str, tags: list[str]
|
||||
) -> Path:
|
||||
"""Helper to create a link artifact."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("", "")):
|
||||
return registry.save_link(url, title=title, tags=tags)
|
||||
|
||||
def test_search_by_url(self, registry: LinkRegistry) -> None:
|
||||
"""Req 8.6: Search finds links matching URL."""
|
||||
self._create_link(registry, "https://python.org", "Python", ["lang"])
|
||||
self._create_link(registry, "https://rust-lang.org", "Rust", ["lang"])
|
||||
|
||||
results = registry.search_links("python")
|
||||
assert len(results) == 1
|
||||
assert results[0].metadata.title == "Python"
|
||||
|
||||
def test_search_by_title(self, registry: LinkRegistry) -> None:
|
||||
"""Req 8.6: Search finds links matching title."""
|
||||
self._create_link(registry, "https://a.com", "Django Documentation", ["web"])
|
||||
self._create_link(registry, "https://b.com", "Flask Tutorial", ["web"])
|
||||
|
||||
results = registry.search_links("django")
|
||||
assert len(results) == 1
|
||||
assert results[0].metadata.title == "Django Documentation"
|
||||
|
||||
def test_search_by_tag(self, registry: LinkRegistry) -> None:
|
||||
"""Req 8.6: Search finds links matching tag."""
|
||||
self._create_link(registry, "https://a.com", "A", ["python", "web"])
|
||||
self._create_link(registry, "https://b.com", "B", ["rust", "systems"])
|
||||
|
||||
results = registry.search_links("rust")
|
||||
assert len(results) == 1
|
||||
assert results[0].metadata.title == "B"
|
||||
|
||||
def test_search_case_insensitive(self, registry: LinkRegistry) -> None:
|
||||
"""Search is case-insensitive."""
|
||||
self._create_link(registry, "https://a.com", "Django Docs", ["Python"])
|
||||
|
||||
results = registry.search_links("DJANGO")
|
||||
assert len(results) == 1
|
||||
|
||||
results = registry.search_links("python")
|
||||
assert len(results) == 1
|
||||
|
||||
def test_search_no_results(self, registry: LinkRegistry) -> None:
|
||||
"""Returns empty list when nothing matches."""
|
||||
self._create_link(registry, "https://a.com", "Something", ["tag"])
|
||||
|
||||
results = registry.search_links("nonexistent")
|
||||
assert results == []
|
||||
|
||||
def test_search_empty_links_dir(self, registry: LinkRegistry) -> None:
|
||||
"""Returns empty list when links directory doesn't exist."""
|
||||
results = registry.search_links("anything")
|
||||
assert results == []
|
||||
|
||||
def test_search_multiple_matches(self, registry: LinkRegistry) -> None:
|
||||
"""Returns all matching links."""
|
||||
self._create_link(registry, "https://a.com", "Python Docs", ["python"])
|
||||
self._create_link(registry, "https://b.com", "Python Tutorial", ["tutorial"])
|
||||
|
||||
results = registry.search_links("python")
|
||||
assert len(results) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _fetch_metadata tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFetchMetadata:
|
||||
"""Tests for _fetch_metadata."""
|
||||
|
||||
def test_extracts_title_from_html(self, registry: LinkRegistry) -> None:
|
||||
"""Extracts <title> from HTML response."""
|
||||
html = b"<html><head><title>Test Page Title</title></head><body></body></html>"
|
||||
|
||||
with patch("urllib.request.urlopen") as mock_urlopen:
|
||||
mock_response = mock_urlopen.return_value.__enter__.return_value
|
||||
mock_response.read.return_value = html
|
||||
mock_response.headers.get_content_charset.return_value = "utf-8"
|
||||
|
||||
title, desc = registry._fetch_metadata("https://example.com")
|
||||
|
||||
assert title == "Test Page Title"
|
||||
|
||||
def test_extracts_meta_description(self, registry: LinkRegistry) -> None:
|
||||
"""Extracts meta description from HTML."""
|
||||
html = (
|
||||
b'<html><head><title>T</title>'
|
||||
b'<meta name="description" content="A page about testing">'
|
||||
b"</head></html>"
|
||||
)
|
||||
|
||||
with patch("urllib.request.urlopen") as mock_urlopen:
|
||||
mock_response = mock_urlopen.return_value.__enter__.return_value
|
||||
mock_response.read.return_value = html
|
||||
mock_response.headers.get_content_charset.return_value = "utf-8"
|
||||
|
||||
title, desc = registry._fetch_metadata("https://example.com")
|
||||
|
||||
assert desc == "A page about testing"
|
||||
|
||||
def test_returns_empty_on_network_error(self, registry: LinkRegistry) -> None:
|
||||
"""Returns empty strings when fetch fails."""
|
||||
with patch("urllib.request.urlopen", side_effect=Exception("Network error")):
|
||||
title, desc = registry._fetch_metadata("https://unreachable.test")
|
||||
|
||||
assert title == ""
|
||||
assert desc == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _find_existing tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFindExisting:
|
||||
"""Tests for _find_existing."""
|
||||
|
||||
def test_finds_existing_url(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Finds an existing link by URL."""
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("T", "")):
|
||||
saved = registry.save_link("https://example.com/exists", title="T")
|
||||
|
||||
found = registry._find_existing("https://example.com/exists")
|
||||
assert found == saved
|
||||
|
||||
def test_returns_none_for_unknown_url(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Returns None when URL doesn't exist."""
|
||||
found = registry._find_existing("https://never-saved.com")
|
||||
assert found is None
|
||||
|
||||
def test_returns_none_for_empty_dir(
|
||||
self, registry: LinkRegistry, context_path: Path
|
||||
) -> None:
|
||||
"""Returns None when links dir doesn't exist."""
|
||||
found = registry._find_existing("https://example.com")
|
||||
assert found is None
|
||||
@@ -0,0 +1,418 @@
|
||||
"""Property-Based Tests für Link Registry.
|
||||
|
||||
**Validates: Requirements 8.1, 8.2, 8.6, 8.7**
|
||||
|
||||
Property 12: Link Artifact Structure
|
||||
*For any* URL saved via the Link Registry, the resulting artifact SHALL have
|
||||
`type: link`, be stored in the `links/` subfolder, and its frontmatter SHALL
|
||||
contain the fields: url, title (non-empty), tags (list), and created (valid date).
|
||||
|
||||
Property 13: Link Deduplication with Tag Merging
|
||||
*For any* URL that is saved twice to the same context Link Registry (possibly
|
||||
with different tags), only one artifact file SHALL exist for that URL, and its
|
||||
tags SHALL be the union of all tags provided across both save operations.
|
||||
|
||||
Property 14: Link Search Completeness
|
||||
*For any* stored link artifact and any substring query that matches its URL,
|
||||
title, or any of its tags, the Link Registry search SHALL return that artifact
|
||||
in its results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, assume, HealthCheck
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.sources.link import LinkRegistry
|
||||
from monorepo.knowledge.artifact import parse_frontmatter
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# URL domain segments (ASCII alphanumeric for reliable filesystem behavior)
|
||||
_domain_segment_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("Ll", "Nd")),
|
||||
min_size=2,
|
||||
max_size=10,
|
||||
)
|
||||
|
||||
# URL path segments
|
||||
_path_segment_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Nd"),
|
||||
whitelist_characters="-_",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=12,
|
||||
)
|
||||
|
||||
|
||||
@st.composite
|
||||
def url_st(draw: st.DrawFn) -> str:
|
||||
"""Generate a valid HTTP(S) URL."""
|
||||
scheme = draw(st.sampled_from(["http", "https"]))
|
||||
domain_parts = [draw(_domain_segment_st) for _ in range(draw(st.integers(2, 3)))]
|
||||
domain = ".".join(domain_parts)
|
||||
path_depth = draw(st.integers(0, 3))
|
||||
path_parts = [draw(_path_segment_st) for _ in range(path_depth)]
|
||||
path = "/".join(path_parts)
|
||||
if path:
|
||||
return f"{scheme}://{domain}/{path}"
|
||||
return f"{scheme}://{domain}"
|
||||
|
||||
|
||||
# Title: non-empty strings with printable chars
|
||||
title_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "P", "Z"),
|
||||
blacklist_characters="\x00\r\n",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=40,
|
||||
).filter(lambda s: s.strip() != "")
|
||||
|
||||
# Tags: lists of short strings (ASCII-ish to avoid YAML edge cases)
|
||||
_tag_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Nd"),
|
||||
whitelist_characters="-_",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=15,
|
||||
)
|
||||
|
||||
tags_st = st.lists(_tag_st, min_size=0, max_size=5, unique=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock helper: prevent network calls during tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _mock_fetch_metadata(url: str) -> tuple[str, str]:
|
||||
"""Mock metadata fetch that returns empty strings (no network)."""
|
||||
return ("", "")
|
||||
|
||||
|
||||
def _make_isolated_registry() -> tuple[LinkRegistry, Path]:
|
||||
"""Create a LinkRegistry in an isolated temporary directory.
|
||||
|
||||
Returns the registry and the temp path (caller should NOT clean up manually,
|
||||
the tempdir is cleaned up by the OS or at process end).
|
||||
"""
|
||||
td = tempfile.mkdtemp()
|
||||
context_path = Path(td)
|
||||
return LinkRegistry(context_path), context_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 12: Link Artifact Structure
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLinkArtifactStructure:
|
||||
"""**Validates: Requirements 8.1, 8.2**
|
||||
|
||||
Property 12: For any URL saved via the Link Registry, the resulting
|
||||
artifact SHALL have `type: link`, be stored in the `links/` subfolder,
|
||||
and its frontmatter SHALL contain the fields: url, title (non-empty),
|
||||
tags (list), and created (valid date).
|
||||
"""
|
||||
|
||||
@given(url=url_st(), title=title_st, tags=tags_st)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_saved_link_has_correct_structure(
|
||||
self, url: str, title: str, tags: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""Any saved link produces an artifact with type=link in links/ folder."""
|
||||
registry, ctx = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
result_path = registry.save_link(url=url, title=title, tags=tags)
|
||||
|
||||
# Stored in links/ subfolder
|
||||
assert "links" in result_path.parts, (
|
||||
f"Artifact not stored in links/ subfolder: {result_path}"
|
||||
)
|
||||
|
||||
# Parse the artifact and verify structure
|
||||
artifact = parse_frontmatter(result_path)
|
||||
|
||||
# type must be "link"
|
||||
assert artifact.metadata.type == "link", (
|
||||
f"Expected type='link', got '{artifact.metadata.type}'"
|
||||
)
|
||||
|
||||
# URL must be present in source
|
||||
assert artifact.metadata.source.get("url") == url, (
|
||||
f"Expected url={url!r} in source, got {artifact.metadata.source}"
|
||||
)
|
||||
|
||||
# Title must be non-empty
|
||||
assert artifact.metadata.title != "", "Title must not be empty"
|
||||
assert artifact.metadata.title.strip() != "", "Title must not be whitespace-only"
|
||||
|
||||
# Tags must be a list
|
||||
assert isinstance(artifact.metadata.tags, list), (
|
||||
f"Tags should be a list, got {type(artifact.metadata.tags)}"
|
||||
)
|
||||
|
||||
# Created must be a valid date
|
||||
assert artifact.metadata.created is not None, "Created date must be set"
|
||||
assert isinstance(artifact.metadata.created, date), (
|
||||
f"Created must be a date, got {type(artifact.metadata.created)}"
|
||||
)
|
||||
|
||||
@given(url=url_st(), tags=tags_st)
|
||||
@settings(max_examples=30, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_saved_link_without_title_still_has_nonempty_title(
|
||||
self, url: str, tags: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""When no title is provided, the artifact still has a non-empty title."""
|
||||
registry, _ = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", return_value=("", "")):
|
||||
result_path = registry.save_link(url=url, title=None, tags=tags)
|
||||
|
||||
artifact = parse_frontmatter(result_path)
|
||||
assert artifact.metadata.title != "", (
|
||||
f"Title should be derived from URL but was empty for {url}"
|
||||
)
|
||||
|
||||
@given(url=url_st(), title=title_st, tags=tags_st)
|
||||
@settings(max_examples=30, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_tags_from_input_are_preserved(
|
||||
self, url: str, title: str, tags: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""All tags provided to save_link appear in the artifact's tags."""
|
||||
registry, _ = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
result_path = registry.save_link(url=url, title=title, tags=tags)
|
||||
|
||||
artifact = parse_frontmatter(result_path)
|
||||
for tag in tags:
|
||||
assert tag in artifact.metadata.tags, (
|
||||
f"Tag {tag!r} missing from artifact tags {artifact.metadata.tags}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 13: Link Deduplication with Tag Merging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLinkDeduplicationWithTagMerging:
|
||||
"""**Validates: Requirements 8.7**
|
||||
|
||||
Property 13: For any URL that is saved twice to the same context Link
|
||||
Registry (possibly with different tags), only one artifact file SHALL
|
||||
exist for that URL, and its tags SHALL be the union of all tags provided
|
||||
across both save operations.
|
||||
"""
|
||||
|
||||
@given(url=url_st(), title=title_st, tags1=tags_st, tags2=tags_st)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_duplicate_url_produces_single_file(
|
||||
self, url: str, title: str, tags1: list[str], tags2: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""Saving same URL twice yields exactly one artifact file."""
|
||||
registry, ctx = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
registry.save_link(url=url, title=title, tags=tags1)
|
||||
registry.save_link(url=url, title=title, tags=tags2)
|
||||
|
||||
# Count files with matching URL in links directory
|
||||
links_dir = ctx / "links"
|
||||
if not links_dir.exists():
|
||||
pytest.fail("links/ directory should exist after saving")
|
||||
|
||||
url_files = []
|
||||
for md_file in links_dir.glob("*.md"):
|
||||
artifact = parse_frontmatter(md_file)
|
||||
if artifact.metadata.source.get("url") == url:
|
||||
url_files.append(md_file)
|
||||
|
||||
assert len(url_files) == 1, (
|
||||
f"Expected exactly 1 file for URL {url!r}, found {len(url_files)}"
|
||||
)
|
||||
|
||||
@given(url=url_st(), title=title_st, tags1=tags_st, tags2=tags_st)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_duplicate_url_merges_tags_as_union(
|
||||
self, url: str, title: str, tags1: list[str], tags2: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""Tags from both save operations form a union in the artifact."""
|
||||
registry, _ = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
registry.save_link(url=url, title=title, tags=tags1)
|
||||
result_path = registry.save_link(url=url, title=title, tags=tags2)
|
||||
|
||||
artifact = parse_frontmatter(result_path)
|
||||
expected_tags = set(tags1) | set(tags2)
|
||||
|
||||
# All tags from both operations must be present
|
||||
for tag in expected_tags:
|
||||
assert tag in artifact.metadata.tags, (
|
||||
f"Tag {tag!r} missing from merged tags. "
|
||||
f"tags1={tags1}, tags2={tags2}, result={artifact.metadata.tags}"
|
||||
)
|
||||
|
||||
@given(url=url_st(), title=title_st, tags=tags_st)
|
||||
@settings(max_examples=30, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_saving_same_tags_twice_does_not_grow_tag_list(
|
||||
self, url: str, title: str, tags: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""Saving same URL with same tags twice does not grow the tag list."""
|
||||
registry, _ = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
registry.save_link(url=url, title=title, tags=tags)
|
||||
result_path = registry.save_link(url=url, title=title, tags=tags)
|
||||
|
||||
artifact = parse_frontmatter(result_path)
|
||||
|
||||
# Tag count should equal the unique set (no duplicates introduced)
|
||||
assert len(artifact.metadata.tags) == len(set(tags)), (
|
||||
f"Expected {len(set(tags))} unique tags but got "
|
||||
f"{len(artifact.metadata.tags)}: {artifact.metadata.tags}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 14: Link Search Completeness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLinkSearchCompleteness:
|
||||
"""**Validates: Requirements 8.6**
|
||||
|
||||
Property 14: For any stored link artifact and any substring query that
|
||||
matches its URL, title, or any of its tags, the Link Registry search
|
||||
SHALL return that artifact in its results.
|
||||
"""
|
||||
|
||||
@given(url=url_st(), title=title_st, tags=tags_st)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_search_by_url_substring_finds_link(
|
||||
self, url: str, title: str, tags: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""Searching by a substring of the URL finds the link."""
|
||||
registry, _ = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
registry.save_link(url=url, title=title, tags=tags)
|
||||
|
||||
# Use domain portion as query (always present after scheme://)
|
||||
parts = url.split("://")
|
||||
assume(len(parts) == 2 and len(parts[1]) >= 4)
|
||||
query = parts[1][:8]
|
||||
assume(len(query) >= 2)
|
||||
|
||||
results = registry.search_links(query)
|
||||
result_urls = [a.metadata.source.get("url", "") for a in results]
|
||||
assert url in result_urls, (
|
||||
f"URL {url!r} not found when searching for {query!r}. "
|
||||
f"Results: {result_urls}"
|
||||
)
|
||||
|
||||
@given(url=url_st(), title=title_st, tags=tags_st)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_search_by_title_substring_finds_link(
|
||||
self, url: str, title: str, tags: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""Searching by a substring of the title finds the link."""
|
||||
registry, _ = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
saved_path = registry.save_link(url=url, title=title, tags=tags)
|
||||
|
||||
# Read back the actual title stored
|
||||
artifact = parse_frontmatter(saved_path)
|
||||
actual_title = artifact.metadata.title
|
||||
assume(len(actual_title) >= 3)
|
||||
|
||||
# Pick a substring of the stored title
|
||||
query = actual_title[:min(8, len(actual_title))]
|
||||
assume(len(query) >= 2)
|
||||
|
||||
results = registry.search_links(query)
|
||||
result_urls = [a.metadata.source.get("url", "") for a in results]
|
||||
assert url in result_urls, (
|
||||
f"URL {url!r} not found when searching by title substring {query!r}. "
|
||||
f"Title stored: {actual_title!r}. Results: {result_urls}"
|
||||
)
|
||||
|
||||
@given(url=url_st(), title=title_st, tags=tags_st)
|
||||
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_search_by_tag_finds_link(
|
||||
self, url: str, title: str, tags: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""Searching by any tag of the link finds it in results."""
|
||||
assume(len(tags) > 0)
|
||||
|
||||
registry, _ = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
registry.save_link(url=url, title=title, tags=tags)
|
||||
|
||||
# Search by first tag
|
||||
query = tags[0]
|
||||
assume(len(query) >= 2)
|
||||
|
||||
results = registry.search_links(query)
|
||||
result_urls = [a.metadata.source.get("url", "") for a in results]
|
||||
assert url in result_urls, (
|
||||
f"URL {url!r} not found when searching by tag {query!r}. "
|
||||
f"Tags: {tags}. Results: {result_urls}"
|
||||
)
|
||||
|
||||
@given(
|
||||
url=url_st(),
|
||||
title=st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N"),
|
||||
whitelist_characters=" -_",
|
||||
blacklist_characters="\x00\r\n",
|
||||
# Restrict to ASCII to avoid Unicode case-folding edge cases
|
||||
# (e.g., µ → Μ → μ where upper(lower(x)) ≠ x)
|
||||
max_codepoint=127,
|
||||
),
|
||||
min_size=3,
|
||||
max_size=30,
|
||||
).filter(lambda s: s.strip() != ""),
|
||||
tags=tags_st,
|
||||
)
|
||||
@settings(max_examples=30, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_search_is_case_insensitive(
|
||||
self, url: str, title: str, tags: list[str], tmp_path: Path
|
||||
) -> None:
|
||||
"""Search is case-insensitive for URL, title, and tags."""
|
||||
registry, _ = _make_isolated_registry()
|
||||
|
||||
with patch.object(registry, "_fetch_metadata", side_effect=_mock_fetch_metadata):
|
||||
saved_path = registry.save_link(url=url, title=title, tags=tags)
|
||||
|
||||
artifact = parse_frontmatter(saved_path)
|
||||
actual_title = artifact.metadata.title
|
||||
assume(len(actual_title) >= 3)
|
||||
|
||||
# Search with upper-cased version of title substring
|
||||
query = actual_title[:5].upper()
|
||||
assume(len(query) >= 2)
|
||||
|
||||
results = registry.search_links(query)
|
||||
result_urls = [a.metadata.source.get("url", "") for a in results]
|
||||
assert url in result_urls, (
|
||||
f"Case-insensitive search failed for query {query!r}. "
|
||||
f"Title: {actual_title!r}. Results: {result_urls}"
|
||||
)
|
||||
@@ -0,0 +1,327 @@
|
||||
"""Tests für MarkdownSource als SourceStrategy-Implementierung.
|
||||
|
||||
Validiert:
|
||||
- SourceStrategy ABC-Interface (extract mit SourceConfig, supports_incremental)
|
||||
- .txt-Dateiunterstützung (Plain-Text ohne Frontmatter)
|
||||
- Inkrementelle Verarbeitung via Content-Hash
|
||||
- Abwärtskompatibilität mit dem alten Interface
|
||||
|
||||
Requirements: 2.2, 2.4, 2.7, 4.1
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceStrategy,
|
||||
)
|
||||
from monorepo.knowledge.sources.markdown import (
|
||||
MarkdownSource,
|
||||
SourceError,
|
||||
_compute_file_hash,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_dir_md(tmp_path: Path) -> Path:
|
||||
"""Verzeichnis mit .md-Dateien mit gültigem Frontmatter."""
|
||||
d = tmp_path / "docs"
|
||||
d.mkdir()
|
||||
|
||||
(d / "note1.md").write_text(
|
||||
"---\ntype: note\ntitle: Note One\ntags: [test]\n---\n# Content One\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(d / "note2.md").write_text(
|
||||
"---\ntype: decision\ntitle: Decision Two\ntags: [arch]\n---\n# Content Two\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return d
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_dir_txt(tmp_path: Path) -> Path:
|
||||
"""Verzeichnis mit .txt-Dateien (ohne Frontmatter)."""
|
||||
d = tmp_path / "texts"
|
||||
d.mkdir()
|
||||
|
||||
(d / "meeting-notes.txt").write_text(
|
||||
"Meeting mit Team A am 2024-01-15\n\n- Punkt 1\n- Punkt 2\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(d / "ideas.txt").write_text(
|
||||
"Neue Ideen für das Projekt.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return d
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_dir_mixed(tmp_path: Path) -> Path:
|
||||
"""Verzeichnis mit .md- und .txt-Dateien."""
|
||||
d = tmp_path / "mixed"
|
||||
d.mkdir()
|
||||
|
||||
(d / "note.md").write_text(
|
||||
"---\ntype: note\ntitle: MD Note\ntags: []\n---\nMarkdown content\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(d / "plain.txt").write_text(
|
||||
"Plain text content\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(d / "readme.rst").write_text(
|
||||
"RST is not supported\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return d
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: SourceStrategy ABC Compliance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSourceStrategyInterface:
|
||||
"""Tests für die SourceStrategy ABC-Konformität."""
|
||||
|
||||
def test_inherits_source_strategy(self) -> None:
|
||||
"""MarkdownSource erbt von SourceStrategy."""
|
||||
assert issubclass(MarkdownSource, SourceStrategy)
|
||||
|
||||
def test_is_instance_of_source_strategy(self) -> None:
|
||||
"""Instanz von MarkdownSource ist auch SourceStrategy-Instanz."""
|
||||
source = MarkdownSource()
|
||||
assert isinstance(source, SourceStrategy)
|
||||
|
||||
def test_supports_incremental_returns_true(self) -> None:
|
||||
"""supports_incremental() gibt True zurück."""
|
||||
source = MarkdownSource()
|
||||
assert source.supports_incremental() is True
|
||||
|
||||
def test_extract_with_source_config(self, source_dir_md: Path) -> None:
|
||||
"""extract(config, context) gibt ExtractionResult zurück."""
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test Markdown",
|
||||
params={"directory": str(source_dir_md)},
|
||||
)
|
||||
source = MarkdownSource()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert isinstance(result, ExtractionResult)
|
||||
assert len(result.artifacts) == 2
|
||||
assert result.errors == []
|
||||
assert result.skipped == 0
|
||||
|
||||
def test_extract_with_config_nonexistent_dir(self, tmp_path: Path) -> None:
|
||||
"""extract(config) mit nicht-existierendem Verzeichnis gibt Fehler."""
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Missing Dir",
|
||||
params={"directory": str(tmp_path / "nonexistent")},
|
||||
)
|
||||
source = MarkdownSource()
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert isinstance(result, ExtractionResult)
|
||||
assert result.artifacts == []
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].retry is True
|
||||
|
||||
def test_extract_without_config_raises_if_no_directory(self) -> None:
|
||||
"""extract() ohne config und ohne directory wirft ValueError."""
|
||||
source = MarkdownSource()
|
||||
with pytest.raises(ValueError, match="Kein Verzeichnis"):
|
||||
source.extract()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: .txt-Unterstützung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTxtSupport:
|
||||
"""Tests für .txt-Dateiverarbeitung."""
|
||||
|
||||
def test_extract_txt_files(self, source_dir_txt: Path) -> None:
|
||||
"""extract() findet und verarbeitet .txt-Dateien."""
|
||||
source = MarkdownSource(directory=source_dir_txt)
|
||||
artifacts = source.extract()
|
||||
|
||||
assert len(artifacts) == 2
|
||||
assert source.errors == []
|
||||
|
||||
def test_txt_artifact_has_minimal_metadata(self, source_dir_txt: Path) -> None:
|
||||
"""Aus .txt-Dateien erstellte Artefakte haben minimales Metadata."""
|
||||
source = MarkdownSource(directory=source_dir_txt)
|
||||
artifacts = source.extract()
|
||||
|
||||
# Finde das ideas.txt-Artefakt
|
||||
ideas = [a for a in artifacts if "ideas" in str(a.file_path).lower()]
|
||||
assert len(ideas) == 1
|
||||
|
||||
artifact = ideas[0]
|
||||
assert artifact.metadata.type == "note"
|
||||
assert artifact.metadata.title == "Ideas"
|
||||
assert artifact.metadata.source["type"] == "file"
|
||||
assert "ideas.txt" in artifact.metadata.source["file"]
|
||||
|
||||
def test_txt_artifact_content_is_file_content(
|
||||
self, source_dir_txt: Path
|
||||
) -> None:
|
||||
"""Der Content des Artefakts ist der Dateiinhalt."""
|
||||
source = MarkdownSource(directory=source_dir_txt)
|
||||
artifacts = source.extract()
|
||||
|
||||
meeting_notes = [
|
||||
a for a in artifacts if "meeting" in str(a.file_path).lower()
|
||||
]
|
||||
assert len(meeting_notes) == 1
|
||||
|
||||
artifact = meeting_notes[0]
|
||||
assert "Meeting mit Team A" in artifact.content
|
||||
|
||||
def test_mixed_md_and_txt(self, source_dir_mixed: Path) -> None:
|
||||
"""extract() verarbeitet .md und .txt, ignoriert .rst."""
|
||||
source = MarkdownSource(directory=source_dir_mixed)
|
||||
artifacts = source.extract()
|
||||
|
||||
assert len(artifacts) == 2
|
||||
extensions = {a.file_path.suffix.lower() for a in artifacts}
|
||||
assert extensions == {".md", ".txt"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Inkrementelle Verarbeitung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIncrementalProcessing:
|
||||
"""Tests für inkrementelle Verarbeitung via Content-Hash."""
|
||||
|
||||
def test_set_known_hashes_skips_unchanged(
|
||||
self, source_dir_md: Path
|
||||
) -> None:
|
||||
"""Dateien mit bekanntem Hash werden übersprungen."""
|
||||
source = MarkdownSource(directory=source_dir_md)
|
||||
|
||||
# Ersten Durchlauf: alle Dateien verarbeiten
|
||||
artifacts_first = source.extract()
|
||||
assert len(artifacts_first) == 2
|
||||
|
||||
# Hashes berechnen
|
||||
hashes: dict[str, str] = {}
|
||||
for f in source_dir_md.rglob("*.md"):
|
||||
hashes[str(f)] = _compute_file_hash(f)
|
||||
|
||||
# Zweiter Durchlauf mit bekannten Hashes
|
||||
source2 = MarkdownSource(directory=source_dir_md)
|
||||
source2.set_known_hashes(hashes)
|
||||
artifacts_second = source2.extract()
|
||||
|
||||
# Alle Dateien sollten übersprungen werden
|
||||
assert len(artifacts_second) == 0
|
||||
|
||||
def test_changed_file_is_reprocessed(self, source_dir_md: Path) -> None:
|
||||
"""Geänderte Dateien werden erneut verarbeitet."""
|
||||
# Hashes vor Änderung
|
||||
hashes: dict[str, str] = {}
|
||||
for f in source_dir_md.rglob("*.md"):
|
||||
hashes[str(f)] = _compute_file_hash(f)
|
||||
|
||||
# Datei ändern
|
||||
note1 = source_dir_md / "note1.md"
|
||||
note1.write_text(
|
||||
"---\ntype: note\ntitle: Note One UPDATED\ntags: [test]\n---\n# Updated\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Extract mit alten Hashes
|
||||
source = MarkdownSource(directory=source_dir_md)
|
||||
source.set_known_hashes(hashes)
|
||||
artifacts = source.extract()
|
||||
|
||||
# Nur die geänderte Datei sollte verarbeitet werden
|
||||
assert len(artifacts) == 1
|
||||
assert artifacts[0].metadata.title == "Note One UPDATED"
|
||||
|
||||
def test_new_interface_reports_skipped_count(
|
||||
self, source_dir_md: Path
|
||||
) -> None:
|
||||
"""ExtractionResult.skipped zählt übersprungene Dateien."""
|
||||
# Hashes berechnen
|
||||
hashes: dict[str, str] = {}
|
||||
for f in source_dir_md.rglob("*.md"):
|
||||
hashes[str(f)] = _compute_file_hash(f)
|
||||
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Test",
|
||||
params={"directory": str(source_dir_md)},
|
||||
)
|
||||
source = MarkdownSource()
|
||||
source.set_known_hashes(hashes)
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert isinstance(result, ExtractionResult)
|
||||
assert result.artifacts == []
|
||||
assert result.skipped == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Legacy-Kompatibilität
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLegacyCompatibility:
|
||||
"""Tests für Abwärtskompatibilität mit dem alten Interface."""
|
||||
|
||||
def test_legacy_constructor(self, source_dir_md: Path) -> None:
|
||||
"""MarkdownSource(directory=...) funktioniert weiterhin."""
|
||||
source = MarkdownSource(directory=source_dir_md)
|
||||
artifacts = source.extract()
|
||||
|
||||
assert isinstance(artifacts, list)
|
||||
assert len(artifacts) == 2
|
||||
|
||||
def test_legacy_errors_attribute(self, tmp_path: Path) -> None:
|
||||
"""self.errors enthält weiterhin SourceError-Instanzen mit file_path/error."""
|
||||
source = MarkdownSource(directory=tmp_path / "nonexistent")
|
||||
source.extract()
|
||||
|
||||
assert len(source.errors) == 1
|
||||
err = source.errors[0]
|
||||
assert isinstance(err, SourceError)
|
||||
assert hasattr(err, "file_path")
|
||||
assert hasattr(err, "error")
|
||||
assert hasattr(err, "retry")
|
||||
assert err.retry is True
|
||||
|
||||
def test_legacy_directory_property(self, source_dir_md: Path) -> None:
|
||||
"""directory-Property ist les- und schreibbar."""
|
||||
source = MarkdownSource(directory=source_dir_md)
|
||||
assert source.directory == source_dir_md
|
||||
|
||||
new_dir = source_dir_md.parent / "other"
|
||||
source.directory = new_dir
|
||||
assert source.directory == new_dir
|
||||
|
||||
def test_legacy_source_error_import(self) -> None:
|
||||
"""SourceError ist aus markdown.py importierbar."""
|
||||
from monorepo.knowledge.sources.markdown import SourceError as SE
|
||||
|
||||
err = SE(file_path="/some/path.md", error="test error")
|
||||
assert err.file_path == "/some/path.md"
|
||||
assert err.error == "test error"
|
||||
assert err.retry is True
|
||||
@@ -0,0 +1,359 @@
|
||||
"""Property-basierte Tests für OrgMyLife Task-Deduplication.
|
||||
|
||||
**Validates: Requirements 10.5, 10.6**
|
||||
|
||||
Property 18: Task Deduplication via Mapping
|
||||
- For any action item whose hash already exists in the `.task-mapping.yaml` file,
|
||||
the OrgMyLife integration SHALL NOT create a new task. Only action items with
|
||||
hashes not present in the mapping SHALL trigger task creation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import yaml
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.ingestion.enrichment import ActionItem
|
||||
from monorepo.knowledge.integrations.orgmylife import (
|
||||
MAPPING_FILENAME,
|
||||
OrgMyLifeIntegration,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@st.composite
|
||||
def action_item_description(draw: st.DrawFn) -> str:
|
||||
"""Generate a valid action item description (non-empty, printable)."""
|
||||
desc = draw(
|
||||
st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "Z", "P"),
|
||||
whitelist_characters=" -_.,!?:;",
|
||||
),
|
||||
min_size=5,
|
||||
max_size=100,
|
||||
).filter(lambda t: t.strip() != "" and len(t.strip()) >= 3)
|
||||
)
|
||||
return desc.strip()
|
||||
|
||||
|
||||
@st.composite
|
||||
def action_item(draw: st.DrawFn) -> ActionItem:
|
||||
"""Generate a valid ActionItem with hash auto-computed."""
|
||||
desc = draw(action_item_description())
|
||||
deadline = draw(st.one_of(
|
||||
st.none(),
|
||||
st.dates().map(lambda d: d.isoformat()),
|
||||
))
|
||||
assignee = draw(st.one_of(
|
||||
st.none(),
|
||||
st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L",)),
|
||||
min_size=3,
|
||||
max_size=20,
|
||||
).filter(lambda t: t.strip() != ""),
|
||||
))
|
||||
return ActionItem(description=desc, assignee=assignee, deadline=deadline)
|
||||
|
||||
|
||||
@st.composite
|
||||
def unique_action_items(draw: st.DrawFn, min_size: int = 1, max_size: int = 10) -> list[ActionItem]:
|
||||
"""Generate a list of action items with unique hashes."""
|
||||
count = draw(st.integers(min_value=min_size, max_value=max_size))
|
||||
items: list[ActionItem] = []
|
||||
seen_hashes: set[str] = set()
|
||||
|
||||
for _ in range(count * 3): # oversample to get enough unique
|
||||
if len(items) >= count:
|
||||
break
|
||||
item = draw(action_item())
|
||||
if item.hash and item.hash not in seen_hashes:
|
||||
seen_hashes.add(item.hash)
|
||||
items.append(item)
|
||||
|
||||
# If we couldn't generate enough unique items, that's fine –
|
||||
# just use what we have (at least 1)
|
||||
if not items:
|
||||
item = ActionItem(description="fallback action item")
|
||||
items.append(item)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_id(draw: st.DrawFn) -> str:
|
||||
"""Generate a valid artifact identifier."""
|
||||
context = draw(st.sampled_from(["bahn", "dhive", "privat"]))
|
||||
category = draw(st.sampled_from(["meeting", "decision", "project", "reference"]))
|
||||
slug = draw(
|
||||
st.from_regex(r"[a-z][a-z0-9\-]{2,15}[a-z0-9]", fullmatch=True)
|
||||
)
|
||||
return f"{context}/{category}/{slug}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_task_id_counter = 0
|
||||
|
||||
|
||||
def _mock_api_sequential(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
"""Mock API that returns unique sequential task IDs."""
|
||||
global _task_id_counter
|
||||
_task_id_counter += 1
|
||||
return _task_id_counter
|
||||
|
||||
|
||||
def _setup_mapping_file(
|
||||
knowledge_path: Path,
|
||||
art_id: str,
|
||||
items: list[ActionItem],
|
||||
) -> None:
|
||||
"""Write a mapping file pre-populated with the given action items."""
|
||||
entries = []
|
||||
for i, item in enumerate(items):
|
||||
entries.append({
|
||||
"artifact_id": art_id,
|
||||
"action_item_hash": item.hash,
|
||||
"task_id": 1000 + i,
|
||||
"created": "2025-07-01",
|
||||
})
|
||||
|
||||
data = {
|
||||
"version": "1.0",
|
||||
"mappings": entries,
|
||||
}
|
||||
mapping_path = knowledge_path / MAPPING_FILENAME
|
||||
mapping_path.write_text(
|
||||
yaml.dump(data, default_flow_style=False, allow_unicode=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProperty18TaskDeduplication:
|
||||
"""Property 18: Task Deduplication via Mapping.
|
||||
|
||||
**Validates: Requirements 10.5, 10.6**
|
||||
|
||||
For any action item whose hash already exists in the `.task-mapping.yaml`
|
||||
file, the OrgMyLife integration SHALL NOT create a new task. Only action
|
||||
items with hashes not present in the mapping SHALL trigger task creation.
|
||||
"""
|
||||
|
||||
@given(
|
||||
art_id=artifact_id(),
|
||||
existing_items=unique_action_items(min_size=1, max_size=5),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_existing_hashes_never_create_tasks(
|
||||
self,
|
||||
art_id: str,
|
||||
existing_items: list[ActionItem],
|
||||
tmp_path_factory,
|
||||
) -> None:
|
||||
"""Action items already in mapping SHALL NOT create new tasks.
|
||||
|
||||
**Validates: Requirements 10.5**
|
||||
|
||||
Given a mapping file containing N action item hashes,
|
||||
when create_tasks is called with those same action items,
|
||||
then zero new tasks are created and the API is never called.
|
||||
"""
|
||||
knowledge_path = tmp_path_factory.mktemp("knowledge")
|
||||
integration = OrgMyLifeIntegration(context_path=knowledge_path, enabled=True)
|
||||
|
||||
# Pre-populate mapping with ALL items
|
||||
_setup_mapping_file(knowledge_path, art_id, existing_items)
|
||||
|
||||
api_call_count = 0
|
||||
|
||||
def counting_api(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
nonlocal api_call_count
|
||||
api_call_count += 1
|
||||
return api_call_count
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=counting_api):
|
||||
result = integration.create_tasks(
|
||||
artifact_id=art_id,
|
||||
action_items=existing_items,
|
||||
source_path="test/path.md",
|
||||
)
|
||||
|
||||
# No new tasks created
|
||||
assert result == [], (
|
||||
f"Expected no new tasks for {len(existing_items)} already-mapped items, "
|
||||
f"but got {len(result)} new mappings"
|
||||
)
|
||||
# API should never have been called
|
||||
assert api_call_count == 0, (
|
||||
f"API was called {api_call_count} time(s) for items that already exist in mapping"
|
||||
)
|
||||
|
||||
@given(
|
||||
art_id=artifact_id(),
|
||||
new_items=unique_action_items(min_size=1, max_size=5),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_new_hashes_always_create_tasks(
|
||||
self,
|
||||
art_id: str,
|
||||
new_items: list[ActionItem],
|
||||
tmp_path_factory,
|
||||
) -> None:
|
||||
"""Action items NOT in mapping SHALL trigger task creation.
|
||||
|
||||
**Validates: Requirements 10.6**
|
||||
|
||||
Given an empty mapping file (or no mapping file),
|
||||
when create_tasks is called with N new action items,
|
||||
then N new tasks are created (one per item).
|
||||
"""
|
||||
knowledge_path = tmp_path_factory.mktemp("knowledge")
|
||||
integration = OrgMyLifeIntegration(context_path=knowledge_path, enabled=True)
|
||||
|
||||
# No pre-existing mapping file → all items are new
|
||||
task_id_seq = iter(range(1, len(new_items) + 1))
|
||||
|
||||
def sequential_api(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
return next(task_id_seq)
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=sequential_api):
|
||||
result = integration.create_tasks(
|
||||
artifact_id=art_id,
|
||||
action_items=new_items,
|
||||
source_path="test/path.md",
|
||||
)
|
||||
|
||||
# All items should produce new mappings
|
||||
assert len(result) == len(new_items), (
|
||||
f"Expected {len(new_items)} new tasks, but got {len(result)}"
|
||||
)
|
||||
|
||||
# Each result maps to the correct action item hash
|
||||
result_hashes = {m.action_item_hash for m in result}
|
||||
expected_hashes = {item.hash for item in new_items}
|
||||
assert result_hashes == expected_hashes, (
|
||||
f"Result hashes {result_hashes} don't match expected {expected_hashes}"
|
||||
)
|
||||
|
||||
@given(
|
||||
art_id=artifact_id(),
|
||||
existing_items=unique_action_items(min_size=1, max_size=4),
|
||||
new_items=unique_action_items(min_size=1, max_size=4),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_mixed_existing_and_new_only_creates_new(
|
||||
self,
|
||||
art_id: str,
|
||||
existing_items: list[ActionItem],
|
||||
new_items: list[ActionItem],
|
||||
tmp_path_factory,
|
||||
) -> None:
|
||||
"""Only items with hashes NOT in mapping trigger creation.
|
||||
|
||||
**Validates: Requirements 10.5, 10.6**
|
||||
|
||||
Given a mapping file with K existing hashes and M new action items
|
||||
(where new items have different hashes from existing ones),
|
||||
when create_tasks is called with both existing + new items,
|
||||
then exactly M new tasks are created.
|
||||
"""
|
||||
knowledge_path = tmp_path_factory.mktemp("knowledge")
|
||||
integration = OrgMyLifeIntegration(context_path=knowledge_path, enabled=True)
|
||||
|
||||
# Ensure new items don't overlap with existing items' hashes
|
||||
existing_hashes = {item.hash for item in existing_items}
|
||||
truly_new = [item for item in new_items if item.hash not in existing_hashes]
|
||||
|
||||
# If all "new" items happen to collide with existing, skip this case
|
||||
if not truly_new:
|
||||
return
|
||||
|
||||
# Pre-populate mapping with existing items only
|
||||
_setup_mapping_file(knowledge_path, art_id, existing_items)
|
||||
|
||||
# Combine: existing (should be skipped) + truly new (should be created)
|
||||
all_items = existing_items + truly_new
|
||||
|
||||
task_id_seq = iter(range(2000, 2000 + len(truly_new) + 1))
|
||||
|
||||
def sequential_api(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
return next(task_id_seq)
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=sequential_api):
|
||||
result = integration.create_tasks(
|
||||
artifact_id=art_id,
|
||||
action_items=all_items,
|
||||
source_path="test/path.md",
|
||||
)
|
||||
|
||||
# Only truly new items should be created
|
||||
assert len(result) == len(truly_new), (
|
||||
f"Expected {len(truly_new)} new tasks (from {len(all_items)} total items, "
|
||||
f"{len(existing_items)} existing), but got {len(result)}"
|
||||
)
|
||||
|
||||
# All created task hashes should be from the truly_new set
|
||||
created_hashes = {m.action_item_hash for m in result}
|
||||
expected_new_hashes = {item.hash for item in truly_new}
|
||||
assert created_hashes == expected_new_hashes, (
|
||||
f"Created hashes {created_hashes} don't match expected new hashes {expected_new_hashes}"
|
||||
)
|
||||
|
||||
@given(
|
||||
art_id=artifact_id(),
|
||||
items=unique_action_items(min_size=1, max_size=5),
|
||||
)
|
||||
@settings(max_examples=30, deadline=None)
|
||||
def test_idempotent_double_run(
|
||||
self,
|
||||
art_id: str,
|
||||
items: list[ActionItem],
|
||||
tmp_path_factory,
|
||||
) -> None:
|
||||
"""Running create_tasks twice yields tasks only on first run.
|
||||
|
||||
**Validates: Requirements 10.5, 10.6**
|
||||
|
||||
For any set of action items, calling create_tasks twice in sequence
|
||||
should create tasks only on the first call. The second call should
|
||||
find all hashes in the mapping and create zero new tasks.
|
||||
"""
|
||||
knowledge_path = tmp_path_factory.mktemp("knowledge")
|
||||
integration = OrgMyLifeIntegration(context_path=knowledge_path, enabled=True)
|
||||
|
||||
call_count = 0
|
||||
|
||||
def counting_api(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return call_count
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=counting_api):
|
||||
first_result = integration.create_tasks(art_id, items, "test.md")
|
||||
second_result = integration.create_tasks(art_id, items, "test.md")
|
||||
|
||||
# First run creates all
|
||||
assert len(first_result) == len(items), (
|
||||
f"First run should create {len(items)} tasks, got {len(first_result)}"
|
||||
)
|
||||
|
||||
# Second run creates none (all hashes now exist in mapping)
|
||||
assert second_result == [], (
|
||||
f"Second run should create 0 tasks (dedup), but got {len(second_result)}"
|
||||
)
|
||||
@@ -0,0 +1,450 @@
|
||||
"""Unit tests for OrgMyLife integration.
|
||||
|
||||
Tests task creation, deduplication via mapping file, deadline handling,
|
||||
and graceful error handling.
|
||||
|
||||
Requirements: 10.1, 10.2, 10.3, 10.4, 10.5, 10.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.ingestion.enrichment import ActionItem
|
||||
from monorepo.knowledge.integrations.orgmylife import (
|
||||
MAPPING_FILENAME,
|
||||
OrgMyLifeIntegration,
|
||||
TaskMapping,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def knowledge_path(tmp_path: Path) -> Path:
|
||||
"""Create a temporary knowledge folder."""
|
||||
knowledge = tmp_path / "knowledge"
|
||||
knowledge.mkdir()
|
||||
return knowledge
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def integration(knowledge_path: Path) -> OrgMyLifeIntegration:
|
||||
"""Create an OrgMyLifeIntegration with mocked API."""
|
||||
return OrgMyLifeIntegration(context_path=knowledge_path, enabled=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_action_items() -> list[ActionItem]:
|
||||
"""Create sample action items for testing."""
|
||||
return [
|
||||
ActionItem(
|
||||
description="API-Konzept bis Freitag finalisieren",
|
||||
assignee="Max Mustermann",
|
||||
deadline="2025-07-04",
|
||||
),
|
||||
ActionItem(
|
||||
description="Testdaten bereitstellen",
|
||||
assignee="Anna Schmidt",
|
||||
deadline=None,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _mock_api_call(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
"""Mock OrgMyLife API that returns sequential task IDs."""
|
||||
# Use hash of title as a pseudo task ID for determinism
|
||||
return abs(hash(title)) % 100000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test: Task creation basics (Req 10.1, 10.2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCreateTasks:
|
||||
"""Tests for basic task creation."""
|
||||
|
||||
def test_creates_tasks_for_action_items(
|
||||
self, integration: OrgMyLifeIntegration, sample_action_items: list[ActionItem]
|
||||
) -> None:
|
||||
"""Tasks should be created for each action item."""
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=_mock_api_call):
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="bahn/meeting/sprint-planning",
|
||||
action_items=sample_action_items,
|
||||
source_path="bahn/knowledge/meetings/sprint-planning.md",
|
||||
)
|
||||
|
||||
assert len(mappings) == 2
|
||||
assert all(isinstance(m, TaskMapping) for m in mappings)
|
||||
|
||||
def test_mapping_contains_correct_artifact_id(
|
||||
self, integration: OrgMyLifeIntegration, sample_action_items: list[ActionItem]
|
||||
) -> None:
|
||||
"""Each mapping should reference the source artifact."""
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=_mock_api_call):
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="bahn/meeting/sprint-planning",
|
||||
action_items=sample_action_items,
|
||||
source_path="bahn/knowledge/meetings/sprint-planning.md",
|
||||
)
|
||||
|
||||
for mapping in mappings:
|
||||
assert mapping.artifact_id == "bahn/meeting/sprint-planning"
|
||||
|
||||
def test_mapping_contains_action_item_hash(
|
||||
self, integration: OrgMyLifeIntegration, sample_action_items: list[ActionItem]
|
||||
) -> None:
|
||||
"""Each mapping should contain the action item's hash."""
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=_mock_api_call):
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="bahn/meeting/sprint-planning",
|
||||
action_items=sample_action_items,
|
||||
source_path="bahn/knowledge/meetings/sprint-planning.md",
|
||||
)
|
||||
|
||||
for i, mapping in enumerate(mappings):
|
||||
assert mapping.action_item_hash == sample_action_items[i].hash
|
||||
|
||||
def test_returns_empty_when_disabled(
|
||||
self, knowledge_path: Path, sample_action_items: list[ActionItem]
|
||||
) -> None:
|
||||
"""Disabled integration should not create any tasks."""
|
||||
integration = OrgMyLifeIntegration(context_path=knowledge_path, enabled=False)
|
||||
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="test-artifact",
|
||||
action_items=sample_action_items,
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
assert mappings == []
|
||||
|
||||
def test_returns_empty_for_empty_action_items(
|
||||
self, integration: OrgMyLifeIntegration
|
||||
) -> None:
|
||||
"""Empty action items list should return no mappings."""
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="test-artifact",
|
||||
action_items=[],
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
assert mappings == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test: Deadline handling (Req 10.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDeadlineHandling:
|
||||
"""Tests for deadline propagation to OrgMyLife tasks."""
|
||||
|
||||
def test_deadline_passed_to_api(
|
||||
self, integration: OrgMyLifeIntegration
|
||||
) -> None:
|
||||
"""Deadline from ActionItem should be passed to the API call."""
|
||||
items = [
|
||||
ActionItem(
|
||||
description="Finish report",
|
||||
deadline="2025-08-15",
|
||||
),
|
||||
]
|
||||
|
||||
calls: list[dict] = []
|
||||
|
||||
def capture_call(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
calls.append({"title": title, "source_url": source_url, "deadline": deadline})
|
||||
return 42
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=capture_call):
|
||||
integration.create_tasks(
|
||||
artifact_id="test-artifact",
|
||||
action_items=items,
|
||||
source_path="path/to/artifact.md",
|
||||
)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["deadline"] == "2025-08-15"
|
||||
assert calls[0]["title"] == "Finish report"
|
||||
assert calls[0]["source_url"] == "path/to/artifact.md"
|
||||
|
||||
def test_none_deadline_passed_when_not_set(
|
||||
self, integration: OrgMyLifeIntegration
|
||||
) -> None:
|
||||
"""None deadline should be passed when ActionItem has no deadline."""
|
||||
items = [ActionItem(description="Do something", deadline=None)]
|
||||
|
||||
calls: list[dict] = []
|
||||
|
||||
def capture_call(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
calls.append({"deadline": deadline})
|
||||
return 99
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=capture_call):
|
||||
integration.create_tasks(
|
||||
artifact_id="test",
|
||||
action_items=items,
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
assert calls[0]["deadline"] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test: Deduplication (Req 10.5, 10.6)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDeduplication:
|
||||
"""Tests for hash-based deduplication via .task-mapping.yaml."""
|
||||
|
||||
def test_does_not_duplicate_existing_tasks(
|
||||
self, integration: OrgMyLifeIntegration, knowledge_path: Path
|
||||
) -> None:
|
||||
"""Already-mapped action items should not be created again."""
|
||||
items = [ActionItem(description="Existing task")]
|
||||
|
||||
# Pre-populate mapping file
|
||||
mapping_data = {
|
||||
"version": "1.0",
|
||||
"mappings": [
|
||||
{
|
||||
"artifact_id": "test-artifact",
|
||||
"action_item_hash": items[0].hash,
|
||||
"task_id": 123,
|
||||
"created": "2025-07-01",
|
||||
},
|
||||
],
|
||||
}
|
||||
mapping_path = knowledge_path / MAPPING_FILENAME
|
||||
mapping_path.write_text(yaml.dump(mapping_data), encoding="utf-8")
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=_mock_api_call):
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="test-artifact",
|
||||
action_items=items,
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
# No new tasks created
|
||||
assert mappings == []
|
||||
|
||||
def test_creates_only_new_action_items(
|
||||
self, integration: OrgMyLifeIntegration, knowledge_path: Path
|
||||
) -> None:
|
||||
"""Only action items not in mapping should create new tasks."""
|
||||
existing_item = ActionItem(description="Already done")
|
||||
new_item = ActionItem(description="Brand new task")
|
||||
|
||||
# Pre-populate with existing item
|
||||
mapping_data = {
|
||||
"version": "1.0",
|
||||
"mappings": [
|
||||
{
|
||||
"artifact_id": "my-artifact",
|
||||
"action_item_hash": existing_item.hash,
|
||||
"task_id": 100,
|
||||
"created": "2025-07-01",
|
||||
},
|
||||
],
|
||||
}
|
||||
mapping_path = knowledge_path / MAPPING_FILENAME
|
||||
mapping_path.write_text(yaml.dump(mapping_data), encoding="utf-8")
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=_mock_api_call):
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="my-artifact",
|
||||
action_items=[existing_item, new_item],
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
# Only the new item should produce a mapping
|
||||
assert len(mappings) == 1
|
||||
assert mappings[0].action_item_hash == new_item.hash
|
||||
|
||||
def test_persists_mapping_after_creation(
|
||||
self, integration: OrgMyLifeIntegration, knowledge_path: Path
|
||||
) -> None:
|
||||
"""New mappings should be persisted to the YAML file."""
|
||||
items = [ActionItem(description="Persist me")]
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", return_value=555):
|
||||
integration.create_tasks(
|
||||
artifact_id="persist-test",
|
||||
action_items=items,
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
# Verify file was written
|
||||
mapping_path = knowledge_path / MAPPING_FILENAME
|
||||
assert mapping_path.exists()
|
||||
|
||||
data = yaml.safe_load(mapping_path.read_text(encoding="utf-8"))
|
||||
assert data["version"] == "1.0"
|
||||
assert len(data["mappings"]) == 1
|
||||
assert data["mappings"][0]["task_id"] == 555
|
||||
assert data["mappings"][0]["artifact_id"] == "persist-test"
|
||||
assert data["mappings"][0]["action_item_hash"] == items[0].hash
|
||||
|
||||
def test_second_run_does_not_duplicate(
|
||||
self, integration: OrgMyLifeIntegration
|
||||
) -> None:
|
||||
"""Running create_tasks twice with same items should only create once."""
|
||||
items = [ActionItem(description="Run twice")]
|
||||
|
||||
call_count = 0
|
||||
|
||||
def counting_api(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return call_count
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=counting_api):
|
||||
first = integration.create_tasks("art-1", items, "test.md")
|
||||
second = integration.create_tasks("art-1", items, "test.md")
|
||||
|
||||
assert len(first) == 1
|
||||
assert len(second) == 0
|
||||
assert call_count == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test: Graceful error handling (Req 10.4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGracefulErrorHandling:
|
||||
"""Tests for graceful API error handling."""
|
||||
|
||||
def test_api_error_logs_warning_and_continues(
|
||||
self, integration: OrgMyLifeIntegration, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""API failures should be logged as warnings, not raised."""
|
||||
items = [
|
||||
ActionItem(description="Task that fails"),
|
||||
ActionItem(description="Task that succeeds"),
|
||||
]
|
||||
|
||||
call_count = 0
|
||||
|
||||
def failing_then_ok(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
raise ConnectionError("OrgMyLife API timeout")
|
||||
return 42
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=failing_then_ok):
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="error-test",
|
||||
action_items=items,
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
# Only the second task should succeed
|
||||
assert len(mappings) == 1
|
||||
assert mappings[0].action_item_hash == items[1].hash
|
||||
|
||||
def test_all_api_failures_returns_empty(
|
||||
self, integration: OrgMyLifeIntegration
|
||||
) -> None:
|
||||
"""If all API calls fail, return empty list (no crash)."""
|
||||
items = [
|
||||
ActionItem(description="Fails 1"),
|
||||
ActionItem(description="Fails 2"),
|
||||
]
|
||||
|
||||
def always_fails(title: str, source_url: str, deadline: str | None = None) -> int:
|
||||
raise RuntimeError("Service unavailable")
|
||||
|
||||
with patch.object(integration, "_call_orgmylife_api", side_effect=always_fails):
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="all-fail",
|
||||
action_items=items,
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
assert mappings == []
|
||||
|
||||
def test_default_api_raises_not_implemented(
|
||||
self, integration: OrgMyLifeIntegration
|
||||
) -> None:
|
||||
"""Without mocking, the API stub should raise NotImplementedError."""
|
||||
items = [ActionItem(description="Test without mock")]
|
||||
|
||||
# The integration should handle the error gracefully
|
||||
mappings = integration.create_tasks(
|
||||
artifact_id="no-mock",
|
||||
action_items=items,
|
||||
source_path="test.md",
|
||||
)
|
||||
|
||||
# Graceful: returns empty, no exception raised
|
||||
assert mappings == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test: Mapping file handling edge cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMappingFileEdgeCases:
|
||||
"""Tests for mapping file read/write edge cases."""
|
||||
|
||||
def test_missing_mapping_file_treated_as_empty(
|
||||
self, integration: OrgMyLifeIntegration, knowledge_path: Path
|
||||
) -> None:
|
||||
"""Non-existent mapping file should be treated as empty mappings."""
|
||||
mapping_path = knowledge_path / MAPPING_FILENAME
|
||||
assert not mapping_path.exists()
|
||||
|
||||
mappings = integration._load_mapping()
|
||||
assert mappings == {}
|
||||
|
||||
def test_corrupted_yaml_treated_as_empty(
|
||||
self, integration: OrgMyLifeIntegration, knowledge_path: Path
|
||||
) -> None:
|
||||
"""Corrupted YAML file should be treated as empty (not crash)."""
|
||||
mapping_path = knowledge_path / MAPPING_FILENAME
|
||||
mapping_path.write_text("{{invalid: yaml: [", encoding="utf-8")
|
||||
|
||||
mappings = integration._load_mapping()
|
||||
assert mappings == {}
|
||||
|
||||
def test_empty_file_treated_as_empty(
|
||||
self, integration: OrgMyLifeIntegration, knowledge_path: Path
|
||||
) -> None:
|
||||
"""Empty mapping file should be treated as empty mappings."""
|
||||
mapping_path = knowledge_path / MAPPING_FILENAME
|
||||
mapping_path.write_text("", encoding="utf-8")
|
||||
|
||||
mappings = integration._load_mapping()
|
||||
assert mappings == {}
|
||||
|
||||
def test_mapping_with_invalid_entries_skips_them(
|
||||
self, integration: OrgMyLifeIntegration, knowledge_path: Path
|
||||
) -> None:
|
||||
"""Invalid entries in mapping file should be skipped."""
|
||||
mapping_data = {
|
||||
"version": "1.0",
|
||||
"mappings": [
|
||||
{"artifact_id": "good", "action_item_hash": "abc123", "task_id": 1, "created": "2025-01-01"},
|
||||
{"bad_entry": True}, # missing required fields
|
||||
{"artifact_id": "also-good", "action_item_hash": "def456", "task_id": 2, "created": "2025-01-02"},
|
||||
],
|
||||
}
|
||||
mapping_path = knowledge_path / MAPPING_FILENAME
|
||||
mapping_path.write_text(yaml.dump(mapping_data), encoding="utf-8")
|
||||
|
||||
mappings = integration._load_mapping()
|
||||
assert len(mappings) == 2
|
||||
@@ -0,0 +1,498 @@
|
||||
"""Unit-Tests für die PDF- und OCR-Quellstrategie.
|
||||
|
||||
Testet PDFSource mit verschiedenen Szenarien:
|
||||
- PDF mit eingebettetem Text
|
||||
- Bilddateien (direkt OCR)
|
||||
- OCR-Fallback bei gescannten PDFs
|
||||
- Fehlerbehandlung (fehlende Bibliothek, ungültige Dateien)
|
||||
- source.file-Referenz in Artefakt-Metadaten
|
||||
|
||||
Requirements: 4.2, 4.3, 4.4, 4.5, 4.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
from monorepo.knowledge.sources.pdf import (
|
||||
IMAGE_EXTENSIONS,
|
||||
OCR_LANGUAGES,
|
||||
PDF_EXTENSIONS,
|
||||
PDFSource,
|
||||
SUPPORTED_EXTENSIONS,
|
||||
_extract_pdf_text,
|
||||
_perform_ocr,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pdf_source() -> PDFSource:
|
||||
"""Erstellt eine PDFSource-Instanz."""
|
||||
return PDFSource()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_config(tmp_path: Path) -> SourceConfig:
|
||||
"""Erstellt eine SourceConfig mit temporärem Verzeichnis."""
|
||||
return SourceConfig(
|
||||
type="file",
|
||||
name="Test PDF Source",
|
||||
params={"path": str(tmp_path)},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_pdf(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine Dummy-PDF-Datei (Binärdaten)."""
|
||||
pdf_file = tmp_path / "test-document.pdf"
|
||||
# Minimales PDF-Platzhalter (wird für Mock-Tests gebraucht)
|
||||
pdf_file.write_bytes(b"%PDF-1.4 dummy content")
|
||||
return pdf_file
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_image(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine Dummy-Bilddatei."""
|
||||
img_file = tmp_path / "test-image.png"
|
||||
img_file.write_bytes(b"\x89PNG\r\n\x1a\n dummy image")
|
||||
return img_file
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_jpg(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine Dummy-JPG-Datei."""
|
||||
img_file = tmp_path / "whiteboard-photo.jpg"
|
||||
img_file.write_bytes(b"\xff\xd8\xff\xe0 dummy jpg")
|
||||
return img_file
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Unterstützte Formate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSupportedExtensions:
|
||||
"""Tests für Dateiformat-Erkennung."""
|
||||
|
||||
def test_pdf_extension_supported(self) -> None:
|
||||
assert ".pdf" in SUPPORTED_EXTENSIONS
|
||||
|
||||
def test_png_extension_supported(self) -> None:
|
||||
assert ".png" in SUPPORTED_EXTENSIONS
|
||||
|
||||
def test_jpg_extension_supported(self) -> None:
|
||||
assert ".jpg" in SUPPORTED_EXTENSIONS
|
||||
|
||||
def test_jpeg_extension_supported(self) -> None:
|
||||
assert ".jpeg" in SUPPORTED_EXTENSIONS
|
||||
|
||||
def test_pdf_and_image_sets_disjoint(self) -> None:
|
||||
assert PDF_EXTENSIONS.isdisjoint(IMAGE_EXTENSIONS)
|
||||
|
||||
def test_ocr_languages_configured(self) -> None:
|
||||
assert OCR_LANGUAGES == "deu+eng"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: PDFSource Interface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPDFSourceInterface:
|
||||
"""Tests für das SourceStrategy-Interface."""
|
||||
|
||||
def test_supports_incremental(self, pdf_source: PDFSource) -> None:
|
||||
assert pdf_source.supports_incremental() is True
|
||||
|
||||
def test_extract_empty_directory(
|
||||
self, pdf_source: PDFSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Leeres Verzeichnis liefert keine Artefakte."""
|
||||
result = pdf_source.extract(source_config, "bahn")
|
||||
assert result.artifacts == []
|
||||
assert result.errors == []
|
||||
|
||||
def test_extract_missing_path_param(self, pdf_source: PDFSource) -> None:
|
||||
"""Fehlender path-Parameter erzeugt einen Fehler."""
|
||||
config = SourceConfig(type="file", name="Bad Config", params={})
|
||||
result = pdf_source.extract(config, "bahn")
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "parse"
|
||||
|
||||
def test_extract_nonexistent_path(self, pdf_source: PDFSource) -> None:
|
||||
"""Nicht-existenter Pfad erzeugt einen Fehler."""
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="Missing Path",
|
||||
params={"path": "/nonexistent/path/to/files"},
|
||||
)
|
||||
result = pdf_source.extract(config, "bahn")
|
||||
assert len(result.errors) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: PDF-Verarbeitung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPDFProcessing:
|
||||
"""Tests für die PDF-Textextraktion."""
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_pdf_with_embedded_text(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_pdf: Path,
|
||||
) -> None:
|
||||
"""PDF mit eingebettetem Text wird korrekt extrahiert."""
|
||||
mock_extract.return_value = ("Dies ist der PDF-Text.", False)
|
||||
|
||||
source_config.params["path"] = str(sample_pdf)
|
||||
result = pdf_source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert "Dies ist der PDF-Text." in artifact.content
|
||||
assert artifact.metadata.ocr_failed is False
|
||||
assert artifact.metadata.source["file"] == str(sample_pdf)
|
||||
assert artifact.metadata.source["type"] == "pdf"
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._perform_ocr")
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_pdf_scanned_with_ocr_success(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
mock_ocr: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_pdf: Path,
|
||||
) -> None:
|
||||
"""Gescanntes PDF: OCR-Fallback erfolgreich."""
|
||||
mock_extract.return_value = ("", True) # Kein Text, OCR benötigt
|
||||
mock_ocr.return_value = ("OCR-extrahierter Text", False)
|
||||
|
||||
source_config.params["path"] = str(sample_pdf)
|
||||
result = pdf_source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert "OCR-extrahierter Text" in artifact.content
|
||||
assert artifact.metadata.ocr_failed is False
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._perform_ocr")
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_pdf_scanned_with_ocr_failure(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
mock_ocr: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_pdf: Path,
|
||||
) -> None:
|
||||
"""Gescanntes PDF: OCR fehlgeschlagen → ocr_failed=True."""
|
||||
mock_extract.return_value = ("", True) # Kein Text, OCR benötigt
|
||||
mock_ocr.return_value = ("", True) # OCR fehlgeschlagen
|
||||
|
||||
source_config.params["path"] = str(sample_pdf)
|
||||
result = pdf_source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.ocr_failed is True
|
||||
assert artifact.content == "# test document\n\n" # Leerer Inhalt
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_pdf_library_not_available(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_pdf: Path,
|
||||
) -> None:
|
||||
"""PDF-Bibliothek fehlt → versucht OCR als Fallback."""
|
||||
mock_extract.side_effect = ImportError("No PDF lib")
|
||||
|
||||
source_config.params["path"] = str(sample_pdf)
|
||||
|
||||
with patch("monorepo.knowledge.sources.pdf._perform_ocr") as mock_ocr:
|
||||
mock_ocr.return_value = ("", True)
|
||||
result = pdf_source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
assert result.artifacts[0].metadata.ocr_failed is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Bilddateien → OCR
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestImageProcessing:
|
||||
"""Tests für die Verarbeitung von Bilddateien."""
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._perform_ocr")
|
||||
def test_image_ocr_success(
|
||||
self,
|
||||
mock_ocr: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_image: Path,
|
||||
) -> None:
|
||||
"""Bilddatei wird via OCR verarbeitet."""
|
||||
mock_ocr.return_value = ("Handschriftlicher Text", False)
|
||||
|
||||
source_config.params["path"] = str(sample_image)
|
||||
result = pdf_source.extract(source_config, "privat")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert "Handschriftlicher Text" in artifact.content
|
||||
assert artifact.metadata.ocr_failed is False
|
||||
assert artifact.metadata.source["type"] == "image"
|
||||
assert artifact.metadata.source["file"] == str(sample_image)
|
||||
assert artifact.metadata.source_context == "privat"
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._perform_ocr")
|
||||
def test_image_ocr_failure(
|
||||
self,
|
||||
mock_ocr: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_image: Path,
|
||||
) -> None:
|
||||
"""Bilddatei: OCR fehlgeschlagen → ocr_failed=True."""
|
||||
mock_ocr.return_value = ("", True)
|
||||
|
||||
source_config.params["path"] = str(sample_image)
|
||||
result = pdf_source.extract(source_config, "privat")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.ocr_failed is True
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._perform_ocr")
|
||||
def test_jpg_processed_as_image(
|
||||
self,
|
||||
mock_ocr: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_jpg: Path,
|
||||
) -> None:
|
||||
"""JPG-Datei wird korrekt als Bild verarbeitet."""
|
||||
mock_ocr.return_value = ("Whiteboard Notiz", False)
|
||||
|
||||
source_config.params["path"] = str(sample_jpg)
|
||||
result = pdf_source.extract(source_config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.source["type"] == "image"
|
||||
assert "whiteboard" in artifact.metadata.title.lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Source-Referenz (source.file)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSourceFileReference:
|
||||
"""Tests für die Beibehaltung der source.file-Referenz.
|
||||
|
||||
Requirements: 4.5 – Referenz auf Originaldatei im Frontmatter.
|
||||
"""
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._perform_ocr")
|
||||
def test_source_file_preserved_for_image(
|
||||
self,
|
||||
mock_ocr: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_image: Path,
|
||||
) -> None:
|
||||
"""source.file enthält den vollständigen Pfad zur Originaldatei."""
|
||||
mock_ocr.return_value = ("Some text", False)
|
||||
|
||||
source_config.params["path"] = str(sample_image)
|
||||
result = pdf_source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert "file" in artifact.metadata.source
|
||||
assert artifact.metadata.source["file"] == str(sample_image)
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_source_file_preserved_for_pdf(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
source_config: SourceConfig,
|
||||
sample_pdf: Path,
|
||||
) -> None:
|
||||
"""source.file enthält den vollständigen Pfad zur Original-PDF."""
|
||||
mock_extract.return_value = ("PDF text", False)
|
||||
|
||||
source_config.params["path"] = str(sample_pdf)
|
||||
result = pdf_source.extract(source_config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert "file" in artifact.metadata.source
|
||||
assert artifact.metadata.source["file"] == str(sample_pdf)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Verzeichnis-Scan
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDirectoryScan:
|
||||
"""Tests für die Dateisammlung aus Verzeichnissen."""
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._perform_ocr")
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_mixed_directory(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
mock_ocr: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Verzeichnis mit gemischten Dateitypen."""
|
||||
# Erstelle Testdateien
|
||||
(tmp_path / "doc.pdf").write_bytes(b"%PDF-1.4 content")
|
||||
(tmp_path / "photo.png").write_bytes(b"\x89PNG data")
|
||||
(tmp_path / "notes.txt").write_text("Text file") # Nicht unterstützt
|
||||
(tmp_path / "readme.md").write_text("# Readme") # Nicht unterstützt
|
||||
|
||||
mock_extract.return_value = ("PDF text", False)
|
||||
mock_ocr.return_value = ("Image text", False)
|
||||
|
||||
config = SourceConfig(
|
||||
type="file", name="Mixed", params={"path": str(tmp_path)}
|
||||
)
|
||||
result = pdf_source.extract(config, "bahn")
|
||||
|
||||
# Nur PDF und PNG werden verarbeitet
|
||||
assert len(result.artifacts) == 2
|
||||
types = {a.metadata.source["type"] for a in result.artifacts}
|
||||
assert "pdf" in types
|
||||
assert "image" in types
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._perform_ocr")
|
||||
def test_subdirectory_scan(
|
||||
self,
|
||||
mock_ocr: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Unterverzeichnisse werden rekursiv durchsucht."""
|
||||
subdir = tmp_path / "subfolder"
|
||||
subdir.mkdir()
|
||||
(subdir / "nested.jpg").write_bytes(b"\xff\xd8 data")
|
||||
|
||||
mock_ocr.return_value = ("Nested text", False)
|
||||
|
||||
config = SourceConfig(
|
||||
type="file", name="Nested", params={"directory": str(tmp_path)}
|
||||
)
|
||||
result = pdf_source.extract(config, "dhive")
|
||||
|
||||
assert len(result.artifacts) == 1
|
||||
assert result.artifacts[0].metadata.source_context == "dhive"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Metadaten
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestArtifactMetadata:
|
||||
"""Tests für die generierten Artefakt-Metadaten."""
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_title_from_filename(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Titel wird aus dem Dateinamen abgeleitet."""
|
||||
pdf_file = tmp_path / "sprint-planning-w24.pdf"
|
||||
pdf_file.write_bytes(b"%PDF content")
|
||||
mock_extract.return_value = ("Planning text", False)
|
||||
|
||||
config = SourceConfig(
|
||||
type="file", name="Test", params={"path": str(pdf_file)}
|
||||
)
|
||||
result = pdf_source.extract(config, "bahn")
|
||||
|
||||
assert result.artifacts[0].metadata.title == "sprint planning w24"
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_content_hash_computed(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Content-Hash wird berechnet."""
|
||||
pdf_file = tmp_path / "doc.pdf"
|
||||
pdf_file.write_bytes(b"%PDF content")
|
||||
mock_extract.return_value = ("Some content", False)
|
||||
|
||||
config = SourceConfig(
|
||||
type="file", name="Test", params={"path": str(pdf_file)}
|
||||
)
|
||||
result = pdf_source.extract(config, "bahn")
|
||||
|
||||
artifact = result.artifacts[0]
|
||||
assert artifact.metadata.content_hash.startswith("sha256:")
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_created_date_set(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Erstelldatum wird auf heute gesetzt."""
|
||||
pdf_file = tmp_path / "doc.pdf"
|
||||
pdf_file.write_bytes(b"%PDF content")
|
||||
mock_extract.return_value = ("Text", False)
|
||||
|
||||
config = SourceConfig(
|
||||
type="file", name="Test", params={"path": str(pdf_file)}
|
||||
)
|
||||
result = pdf_source.extract(config, "bahn")
|
||||
|
||||
assert result.artifacts[0].metadata.created == date.today()
|
||||
|
||||
@patch("monorepo.knowledge.sources.pdf._extract_pdf_text")
|
||||
def test_category_set_to_references(
|
||||
self,
|
||||
mock_extract: MagicMock,
|
||||
pdf_source: PDFSource,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Kategorie ist standardmäßig 'references'."""
|
||||
pdf_file = tmp_path / "doc.pdf"
|
||||
pdf_file.write_bytes(b"%PDF content")
|
||||
mock_extract.return_value = ("Text", False)
|
||||
|
||||
config = SourceConfig(
|
||||
type="file", name="Test", params={"path": str(pdf_file)}
|
||||
)
|
||||
result = pdf_source.extract(config, "bahn")
|
||||
|
||||
assert result.artifacts[0].metadata.category == "references"
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Property-Based Tests für Context Routing und Category Mapping.
|
||||
|
||||
**Validates: Requirements 1.4, 2.3, 3.2, 3.5, 9.6**
|
||||
|
||||
Property 2: Context Derivation from Path
|
||||
*For any* file path that resides under a context folder (bahn/, dhive/, privat/),
|
||||
the context derivation function SHALL return the correct context string matching
|
||||
the top-level folder name.
|
||||
|
||||
Property 6: Context Routing Determinism
|
||||
*For any* valid SourceConfig with an assigned context, the ContextRouter SHALL
|
||||
always route to that context. For any working directory within a context folder,
|
||||
the router SHALL derive the same context as the path-based derivation.
|
||||
|
||||
Property 8: Category to Folder Mapping
|
||||
*For any* valid category string from the set {meeting, decision, project, reference,
|
||||
link, inbox}, the folder mapping function SHALL return the corresponding plural
|
||||
folder name. For any unknown category, it SHALL default to inbox/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings, assume
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.ingestion.router import (
|
||||
CATEGORY_FOLDER_MAP,
|
||||
VALID_CONTEXTS,
|
||||
ContextRouter,
|
||||
category_to_folder,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.config import SourceConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Valid contexts sampled from the canonical set
|
||||
context_st = st.sampled_from(VALID_CONTEXTS)
|
||||
|
||||
# Known categories (the valid set)
|
||||
known_category_st = st.sampled_from(list(CATEGORY_FOLDER_MAP.keys()))
|
||||
|
||||
# Unknown categories: strings that are NOT in the known set
|
||||
unknown_category_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Lu"),
|
||||
whitelist_characters="_-",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=30,
|
||||
).filter(lambda s: s.lower() not in CATEGORY_FOLDER_MAP)
|
||||
|
||||
# Subdirectory path segments (simulating nested paths under a context folder)
|
||||
# IMPORTANT: segments must NOT be valid context names, otherwise the router
|
||||
# will match the wrong context (it scans for /{context}/ in order).
|
||||
path_segment_st = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("Ll", "Lu", "Nd"),
|
||||
whitelist_characters="-_.",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=20,
|
||||
).filter(lambda s: s.lower() not in VALID_CONTEXTS)
|
||||
|
||||
# List of subdirectory segments to form a path suffix
|
||||
path_suffix_st = st.lists(path_segment_st, min_size=0, max_size=4)
|
||||
|
||||
# Monorepo root paths (Unix and Windows style)
|
||||
root_path_st = st.sampled_from([
|
||||
"/home/user/monorepo",
|
||||
"/workspace/project",
|
||||
"C:/Users/user/Coden/Orchestrator",
|
||||
])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 2: Context Derivation from Path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestContextDerivationFromPath:
|
||||
"""**Validates: Requirements 1.4, 3.2**
|
||||
|
||||
Property 2: For any file path under a context folder, determine_context
|
||||
SHALL return the correct context string.
|
||||
"""
|
||||
|
||||
@given(
|
||||
root=root_path_st,
|
||||
context=context_st,
|
||||
subdirs=path_suffix_st,
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_path_under_context_returns_correct_context(
|
||||
self,
|
||||
root: str,
|
||||
context: str,
|
||||
subdirs: list[str],
|
||||
) -> None:
|
||||
"""Any path containing /{context}/ derives to that context."""
|
||||
router = ContextRouter(Path(root))
|
||||
# Build a path like /root/{context}/sub1/sub2/...
|
||||
path_parts = [root, context] + subdirs
|
||||
cwd = Path("/".join(path_parts))
|
||||
result = router.determine_context(cwd=cwd)
|
||||
assert result == context
|
||||
|
||||
@given(
|
||||
root=root_path_st,
|
||||
context=context_st,
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_path_ending_with_context_returns_correct_context(
|
||||
self,
|
||||
root: str,
|
||||
context: str,
|
||||
) -> None:
|
||||
"""A path ending exactly with the context name also resolves correctly."""
|
||||
router = ContextRouter(Path(root))
|
||||
cwd = Path(f"{root}/{context}")
|
||||
result = router.determine_context(cwd=cwd)
|
||||
assert result == context
|
||||
|
||||
@given(
|
||||
root=root_path_st,
|
||||
context=context_st,
|
||||
subdirs=path_suffix_st,
|
||||
filename=path_segment_st,
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_file_path_under_context_derives_context(
|
||||
self,
|
||||
root: str,
|
||||
context: str,
|
||||
subdirs: list[str],
|
||||
filename: str,
|
||||
) -> None:
|
||||
"""Even deeply nested file paths under a context derive correctly."""
|
||||
router = ContextRouter(Path(root))
|
||||
parts = [root, context, "knowledge"] + subdirs + [f"{filename}.md"]
|
||||
file_path = Path("/".join(parts))
|
||||
result = router.determine_context(cwd=file_path)
|
||||
assert result == context
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 6: Context Routing Determinism
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestContextRoutingDeterminism:
|
||||
"""**Validates: Requirements 2.3**
|
||||
|
||||
Property 6: For any valid SourceConfig with an assigned context, the
|
||||
ContextRouter SHALL always route to that context. For any working directory
|
||||
within a context folder, the router SHALL derive the same context as the
|
||||
path-based derivation.
|
||||
"""
|
||||
|
||||
@given(
|
||||
context=context_st,
|
||||
source_type=st.sampled_from(["confluence", "jira", "email", "file", "link"]),
|
||||
source_name=st.text(min_size=1, max_size=20, alphabet="abcdefghijklmnop"),
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_source_config_with_context_always_routes_to_that_context(
|
||||
self,
|
||||
context: str,
|
||||
source_type: str,
|
||||
source_name: str,
|
||||
) -> None:
|
||||
"""SourceConfig.params['context'] always determines the routing outcome."""
|
||||
router = ContextRouter(Path("/monorepo"))
|
||||
config = SourceConfig(
|
||||
type=source_type,
|
||||
name=source_name,
|
||||
params={"context": context},
|
||||
)
|
||||
# Call multiple times to verify determinism
|
||||
result1 = router.determine_context(source_config=config)
|
||||
result2 = router.determine_context(source_config=config)
|
||||
result3 = router.determine_context(source_config=config)
|
||||
assert result1 == context
|
||||
assert result2 == context
|
||||
assert result3 == context
|
||||
|
||||
@given(
|
||||
root=root_path_st,
|
||||
context=context_st,
|
||||
subdirs=path_suffix_st,
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_cwd_derivation_matches_path_based_derivation(
|
||||
self,
|
||||
root: str,
|
||||
context: str,
|
||||
subdirs: list[str],
|
||||
) -> None:
|
||||
"""For any cwd within a context folder, repeated derivation yields the same result."""
|
||||
router = ContextRouter(Path(root))
|
||||
cwd = Path("/".join([root, context] + subdirs))
|
||||
# Path-based derivation via cwd
|
||||
result_cwd = router.determine_context(cwd=cwd)
|
||||
# Derive again (determinism)
|
||||
result_cwd_again = router.determine_context(cwd=cwd)
|
||||
assert result_cwd == context
|
||||
assert result_cwd_again == context
|
||||
|
||||
@given(
|
||||
context=context_st,
|
||||
subdirs=path_suffix_st,
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_source_config_context_matches_cwd_derivation(
|
||||
self,
|
||||
context: str,
|
||||
subdirs: list[str],
|
||||
) -> None:
|
||||
"""SourceConfig context and cwd under same context agree."""
|
||||
router = ContextRouter(Path("/monorepo"))
|
||||
config = SourceConfig(
|
||||
type="file",
|
||||
name="test-source",
|
||||
params={"context": context},
|
||||
)
|
||||
cwd = Path("/".join(["/monorepo", context] + subdirs))
|
||||
|
||||
result_config = router.determine_context(source_config=config)
|
||||
result_cwd = router.determine_context(cwd=cwd)
|
||||
assert result_config == result_cwd == context
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 8: Category to Folder Mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCategoryToFolderMapping:
|
||||
"""**Validates: Requirements 3.5, 9.6**
|
||||
|
||||
Property 8: For any valid category string from the known set, the folder
|
||||
mapping returns the corresponding plural name. For unknown categories, it
|
||||
defaults to inbox/.
|
||||
"""
|
||||
|
||||
@given(category=known_category_st)
|
||||
@settings(max_examples=100)
|
||||
def test_known_categories_map_to_plural_folder(
|
||||
self,
|
||||
category: str,
|
||||
) -> None:
|
||||
"""Every known category maps to its expected plural folder name."""
|
||||
result = category_to_folder(category)
|
||||
expected = CATEGORY_FOLDER_MAP[category]
|
||||
assert result == expected
|
||||
|
||||
@given(category=unknown_category_st)
|
||||
@settings(max_examples=200)
|
||||
def test_unknown_categories_default_to_inbox(
|
||||
self,
|
||||
category: str,
|
||||
) -> None:
|
||||
"""Any category not in the known set defaults to 'inbox'."""
|
||||
result = category_to_folder(category)
|
||||
assert result == "inbox"
|
||||
|
||||
@given(category=known_category_st)
|
||||
@settings(max_examples=100)
|
||||
def test_case_insensitive_known_categories(
|
||||
self,
|
||||
category: str,
|
||||
) -> None:
|
||||
"""Category mapping is case-insensitive for known categories."""
|
||||
result_upper = category_to_folder(category.upper())
|
||||
result_lower = category_to_folder(category.lower())
|
||||
result_title = category_to_folder(category.title())
|
||||
expected = CATEGORY_FOLDER_MAP[category]
|
||||
assert result_upper == expected
|
||||
assert result_lower == expected
|
||||
assert result_title == expected
|
||||
|
||||
@given(
|
||||
category=known_category_st,
|
||||
context=context_st,
|
||||
filename=path_segment_st,
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_resolve_target_path_uses_category_mapping(
|
||||
self,
|
||||
category: str,
|
||||
context: str,
|
||||
filename: str,
|
||||
) -> None:
|
||||
"""resolve_target_path produces path with the mapped folder name."""
|
||||
router = ContextRouter(Path("/monorepo"))
|
||||
result_path = router.resolve_target_path(context, category, f"{filename}.md")
|
||||
expected_folder = CATEGORY_FOLDER_MAP[category]
|
||||
expected_path = Path(f"/monorepo/{context}/knowledge/{expected_folder}/{filename}.md")
|
||||
assert result_path == expected_path
|
||||
@@ -0,0 +1,456 @@
|
||||
"""Unit-Tests für die WissensdatenbankSource.
|
||||
|
||||
Testet den Adapter für bahn/wissensdatenbank/output/ – Read-Only-Referenzen
|
||||
im bahn/knowledge/ YAML-Index ohne Modifikation der Originaldateien.
|
||||
|
||||
Requirements: 13.1, 13.2, 13.3, 13.4, 13.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.sources.base import ExtractionResult, SourceConfig
|
||||
from monorepo.knowledge.sources.wissensdatenbank import (
|
||||
WissensdatenbankSource,
|
||||
_compute_file_hash,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wdb_dir(tmp_path: Path) -> Path:
|
||||
"""Erstellt ein temporäres Wissensdatenbank-Output-Verzeichnis mit Testdaten."""
|
||||
output = tmp_path / "wissensdatenbank" / "output"
|
||||
output.mkdir(parents=True)
|
||||
|
||||
# Einige Markdown-Dateien erstellen
|
||||
(output / "inb-2026-kapitel-1.md").write_text(
|
||||
"# INB 2026 Kapitel 1\n\nInhalt des ersten Kapitels.",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(output / "inb-2026-kapitel-2.md").write_text(
|
||||
"# INB 2026 Kapitel 2\n\nInhalt des zweiten Kapitels.",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Unterverzeichnis mit weiterer Datei
|
||||
sub = output / "themen"
|
||||
sub.mkdir()
|
||||
(sub / "signaltechnik.md").write_text(
|
||||
"# Signaltechnik\n\nBeschreibung der Signaltechnik.",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Eine .txt-Datei
|
||||
(output / "notizen.txt").write_text(
|
||||
"Allgemeine Notizen zur Wissensdatenbank.",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_config(wdb_dir: Path) -> SourceConfig:
|
||||
"""Erstellt eine SourceConfig für die Wissensdatenbank."""
|
||||
return SourceConfig(
|
||||
type="wissensdatenbank",
|
||||
name="Wissensdatenbank",
|
||||
params={"output_path": str(wdb_dir)},
|
||||
target_folder="references",
|
||||
sync_frequency="manual",
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source() -> WissensdatenbankSource:
|
||||
"""Erstellt eine WissensdatenbankSource-Instanz."""
|
||||
return WissensdatenbankSource()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Grundlegende Extraktion (Req 13.1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBasicExtraction:
|
||||
"""Tests für die grundlegende Extraktion von Wissensdatenbank-Artefakten."""
|
||||
|
||||
def test_extracts_all_markdown_files(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Alle .md- und .txt-Dateien im Output-Verzeichnis werden extrahiert."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
assert isinstance(result, ExtractionResult)
|
||||
assert len(result.artifacts) == 4 # 2 .md + 1 subdir .md + 1 .txt
|
||||
assert result.skipped == 0
|
||||
assert len(result.errors) == 0
|
||||
|
||||
def test_extracts_recursively(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Dateien in Unterverzeichnissen werden ebenfalls extrahiert."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
file_paths = [str(a.file_path) for a in result.artifacts]
|
||||
# signaltechnik.md aus themen/ sollte enthalten sein
|
||||
assert any("signaltechnik" in p for p in file_paths)
|
||||
|
||||
def test_ignores_non_supported_extensions(
|
||||
self,
|
||||
source: WissensdatenbankSource,
|
||||
source_config: SourceConfig,
|
||||
wdb_dir: Path,
|
||||
) -> None:
|
||||
"""Dateien mit nicht-unterstützten Endungen werden ignoriert."""
|
||||
(wdb_dir / "bild.png").write_bytes(b"\x89PNG...")
|
||||
(wdb_dir / "dokument.pdf").write_bytes(b"%PDF-1.4")
|
||||
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
# Nur .md und .txt werden verarbeitet
|
||||
assert len(result.artifacts) == 4
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Read-Only-Referenz-Metadaten (Req 13.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReadOnlyMetadata:
|
||||
"""Tests für korrekte source-Metadaten der Wissensdatenbank-Artefakte."""
|
||||
|
||||
def test_source_type_is_wissensdatenbank(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Artefakte haben source.type == 'wissensdatenbank'."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
for artifact in result.artifacts:
|
||||
assert artifact.metadata.source["type"] == "wissensdatenbank"
|
||||
|
||||
def test_source_path_references_original(
|
||||
self,
|
||||
source: WissensdatenbankSource,
|
||||
source_config: SourceConfig,
|
||||
wdb_dir: Path,
|
||||
) -> None:
|
||||
"""Artefakte enthalten den Originalpfad in source.path."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
for artifact in result.artifacts:
|
||||
assert "path" in artifact.metadata.source
|
||||
# Der Pfad sollte auf eine existierende Datei zeigen
|
||||
original_path = Path(artifact.metadata.source["path"])
|
||||
assert original_path.exists()
|
||||
|
||||
def test_source_context_is_bahn(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Artefakte haben source_context == 'bahn' (übergebener Kontext)."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
for artifact in result.artifacts:
|
||||
assert artifact.metadata.source_context == "bahn"
|
||||
|
||||
def test_artifact_type_is_reference(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Artefakte sind vom Typ 'reference'."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
for artifact in result.artifacts:
|
||||
assert artifact.metadata.type == "reference"
|
||||
|
||||
def test_category_is_references(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Artefakte haben category == 'references'."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
for artifact in result.artifacts:
|
||||
assert artifact.metadata.category == "references"
|
||||
|
||||
def test_content_hash_is_set(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Artefakte haben einen nicht-leeren content_hash."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
for artifact in result.artifacts:
|
||||
assert artifact.metadata.content_hash.startswith("sha256:")
|
||||
|
||||
def test_title_derived_from_filename(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Titel wird aus dem Dateinamen abgeleitet."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
titles = [a.metadata.title for a in result.artifacts]
|
||||
assert "Inb 2026 Kapitel 1" in titles
|
||||
assert "Signaltechnik" in titles
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Originaldateien unverändert (Req 13.2, 13.4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReadOnlyBehavior:
|
||||
"""Tests, dass die Originaldateien nicht modifiziert werden."""
|
||||
|
||||
def test_original_files_unchanged(
|
||||
self,
|
||||
source: WissensdatenbankSource,
|
||||
source_config: SourceConfig,
|
||||
wdb_dir: Path,
|
||||
) -> None:
|
||||
"""Originaldateien werden nicht verändert – gleicher Hash vor/nach Extraktion."""
|
||||
# Hashes vor Extraktion berechnen
|
||||
files = list(wdb_dir.rglob("*"))
|
||||
hashes_before = {
|
||||
str(f): _compute_file_hash(f) for f in files if f.is_file()
|
||||
}
|
||||
|
||||
# Extraktion ausführen
|
||||
source.extract(source_config, "bahn")
|
||||
|
||||
# Hashes nach Extraktion prüfen
|
||||
hashes_after = {
|
||||
str(f): _compute_file_hash(f) for f in files if f.is_file()
|
||||
}
|
||||
assert hashes_before == hashes_after
|
||||
|
||||
def test_no_new_files_created_in_source_dir(
|
||||
self,
|
||||
source: WissensdatenbankSource,
|
||||
source_config: SourceConfig,
|
||||
wdb_dir: Path,
|
||||
) -> None:
|
||||
"""Keine neuen Dateien werden im Quellverzeichnis erstellt."""
|
||||
files_before = set(str(f) for f in wdb_dir.rglob("*") if f.is_file())
|
||||
|
||||
source.extract(source_config, "bahn")
|
||||
|
||||
files_after = set(str(f) for f in wdb_dir.rglob("*") if f.is_file())
|
||||
assert files_before == files_after
|
||||
|
||||
def test_artifact_file_path_points_to_original(
|
||||
self, source: WissensdatenbankSource, source_config: SourceConfig
|
||||
) -> None:
|
||||
"""Artifact.file_path zeigt auf die Originaldatei (Read-Only-Referenz)."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
for artifact in result.artifacts:
|
||||
# file_path zeigt auf die originale Datei
|
||||
assert artifact.file_path.exists()
|
||||
assert artifact.file_path.suffix.lower() in {".md", ".txt"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Inkrementelle Verarbeitung (Req 13.2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIncrementalProcessing:
|
||||
"""Tests für inkrementelle Verarbeitung via Content-Hash."""
|
||||
|
||||
def test_supports_incremental(
|
||||
self, source: WissensdatenbankSource
|
||||
) -> None:
|
||||
"""WissensdatenbankSource unterstützt inkrementelle Verarbeitung."""
|
||||
assert source.supports_incremental() is True
|
||||
|
||||
def test_skips_unchanged_files(
|
||||
self,
|
||||
source: WissensdatenbankSource,
|
||||
source_config: SourceConfig,
|
||||
wdb_dir: Path,
|
||||
) -> None:
|
||||
"""Dateien mit bekanntem Hash werden übersprungen."""
|
||||
# Erste Extraktion – alles neu
|
||||
result1 = source.extract(source_config, "bahn")
|
||||
assert result1.skipped == 0
|
||||
assert len(result1.artifacts) == 4
|
||||
|
||||
# Hashes der extrahierten Dateien merken
|
||||
known = {}
|
||||
for f in wdb_dir.rglob("*"):
|
||||
if f.is_file() and f.suffix.lower() in {".md", ".txt"}:
|
||||
known[str(f)] = _compute_file_hash(f)
|
||||
|
||||
source.set_known_hashes(known)
|
||||
|
||||
# Zweite Extraktion – alles unverändert → alles geskippt
|
||||
result2 = source.extract(source_config, "bahn")
|
||||
assert result2.skipped == 4
|
||||
assert len(result2.artifacts) == 0
|
||||
|
||||
def test_processes_changed_files(
|
||||
self,
|
||||
source: WissensdatenbankSource,
|
||||
source_config: SourceConfig,
|
||||
wdb_dir: Path,
|
||||
) -> None:
|
||||
"""Geänderte Dateien werden erneut verarbeitet."""
|
||||
# Erste Extraktion
|
||||
source.extract(source_config, "bahn")
|
||||
|
||||
# Hashes merken
|
||||
known = {}
|
||||
for f in wdb_dir.rglob("*"):
|
||||
if f.is_file() and f.suffix.lower() in {".md", ".txt"}:
|
||||
known[str(f)] = _compute_file_hash(f)
|
||||
|
||||
source.set_known_hashes(known)
|
||||
|
||||
# Eine Datei ändern
|
||||
changed_file = wdb_dir / "inb-2026-kapitel-1.md"
|
||||
changed_file.write_text(
|
||||
"# INB 2026 Kapitel 1 (aktualisiert)\n\nNeuer Inhalt.",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Zweite Extraktion – nur die geänderte Datei wird verarbeitet
|
||||
result = source.extract(source_config, "bahn")
|
||||
assert len(result.artifacts) == 1
|
||||
assert result.skipped == 3
|
||||
assert "inb-2026-kapitel-1" in str(result.artifacts[0].file_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Fehlerbehandlung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestErrorHandling:
|
||||
"""Tests für die Fehlerbehandlung."""
|
||||
|
||||
def test_missing_output_path_param(
|
||||
self, source: WissensdatenbankSource
|
||||
) -> None:
|
||||
"""Fehlende output_path-Konfiguration erzeugt einen Fehler."""
|
||||
config = SourceConfig(
|
||||
type="wissensdatenbank",
|
||||
name="Test",
|
||||
params={},
|
||||
)
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.errors) == 1
|
||||
assert "output_path" in result.errors[0].message
|
||||
|
||||
def test_nonexistent_directory(
|
||||
self, source: WissensdatenbankSource, tmp_path: Path
|
||||
) -> None:
|
||||
"""Nicht-existierendes Verzeichnis erzeugt einen Fehler."""
|
||||
config = SourceConfig(
|
||||
type="wissensdatenbank",
|
||||
name="Test",
|
||||
params={"output_path": str(tmp_path / "nonexistent")},
|
||||
)
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "connection"
|
||||
assert result.errors[0].retry is True
|
||||
|
||||
def test_path_is_file_not_directory(
|
||||
self, source: WissensdatenbankSource, tmp_path: Path
|
||||
) -> None:
|
||||
"""Pfad zu einer Datei (statt Verzeichnis) erzeugt einen Fehler."""
|
||||
file = tmp_path / "file.md"
|
||||
file.write_text("content", encoding="utf-8")
|
||||
|
||||
config = SourceConfig(
|
||||
type="wissensdatenbank",
|
||||
name="Test",
|
||||
params={"output_path": str(file)},
|
||||
)
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].error_type == "parse"
|
||||
assert result.errors[0].retry is False
|
||||
|
||||
def test_empty_directory(
|
||||
self, source: WissensdatenbankSource, tmp_path: Path
|
||||
) -> None:
|
||||
"""Leeres Verzeichnis erzeugt leeres Ergebnis ohne Fehler."""
|
||||
empty_dir = tmp_path / "empty"
|
||||
empty_dir.mkdir()
|
||||
|
||||
config = SourceConfig(
|
||||
type="wissensdatenbank",
|
||||
name="Test",
|
||||
params={"output_path": str(empty_dir)},
|
||||
)
|
||||
result = source.extract(config, "bahn")
|
||||
|
||||
assert len(result.artifacts) == 0
|
||||
assert len(result.errors) == 0
|
||||
assert result.skipped == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Kein OKF-Reformat (Req 13.4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNoOkfReformat:
|
||||
"""Tests, dass kein Google-OKF-Reformat erzwungen wird."""
|
||||
|
||||
def test_content_preserved_exactly(
|
||||
self,
|
||||
source: WissensdatenbankSource,
|
||||
source_config: SourceConfig,
|
||||
wdb_dir: Path,
|
||||
) -> None:
|
||||
"""Dateiinhalte werden unverändert als Content übernommen."""
|
||||
result = source.extract(source_config, "bahn")
|
||||
|
||||
# Finde das Artefakt für kapitel-1
|
||||
kapitel1 = next(
|
||||
a for a in result.artifacts if "kapitel-1" in str(a.file_path)
|
||||
)
|
||||
expected_content = "# INB 2026 Kapitel 1\n\nInhalt des ersten Kapitels."
|
||||
assert kapitel1.content == expected_content
|
||||
|
||||
def test_no_frontmatter_injection_in_original(
|
||||
self,
|
||||
source: WissensdatenbankSource,
|
||||
source_config: SourceConfig,
|
||||
wdb_dir: Path,
|
||||
) -> None:
|
||||
"""Originaldateien erhalten KEIN YAML-Frontmatter."""
|
||||
source.extract(source_config, "bahn")
|
||||
|
||||
# Originaldatei prüfen – darf kein '---' am Anfang haben
|
||||
original = wdb_dir / "inb-2026-kapitel-1.md"
|
||||
content = original.read_text(encoding="utf-8")
|
||||
assert not content.startswith("---")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Import von __init__.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestModuleExport:
|
||||
"""Tests, dass WissensdatenbankSource aus dem Modul exportiert wird."""
|
||||
|
||||
def test_importable_from_sources_package(self) -> None:
|
||||
"""WissensdatenbankSource ist über das sources-Package importierbar."""
|
||||
from monorepo.knowledge.sources import WissensdatenbankSource as WDB
|
||||
|
||||
assert WDB is WissensdatenbankSource
|
||||
Reference in New Issue
Block a user