feat: implement knowledge management system (spec complete, all 53 tasks done)
This commit is contained in:
@@ -680,6 +680,10 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
# --- uninstall-hooks ---
|
||||
subparsers.add_parser("uninstall-hooks", help="Git-Hooks entfernen")
|
||||
|
||||
# --- knowledge (Subcommand-Gruppe) ---
|
||||
from monorepo.knowledge.cli import register_knowledge_subcommand
|
||||
register_knowledge_subcommand(subparsers)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
@@ -703,6 +707,7 @@ _COMMAND_HANDLERS = {
|
||||
"fed-status": _cmd_fed_status,
|
||||
"install-hooks": _cmd_install_hooks,
|
||||
"uninstall-hooks": _cmd_uninstall_hooks,
|
||||
"knowledge": "_knowledge", # Sentinel – handled specially in main()
|
||||
}
|
||||
|
||||
|
||||
@@ -722,6 +727,13 @@ def main() -> None:
|
||||
parser.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
# Special handling for knowledge subcommand (has sub-subparsers)
|
||||
if args.command == "knowledge":
|
||||
from monorepo.knowledge.cli import handle_knowledge_command
|
||||
root = _find_monorepo_root()
|
||||
exit_code = handle_knowledge_command(args, root)
|
||||
sys.exit(exit_code)
|
||||
|
||||
handler = _COMMAND_HANDLERS.get(args.command)
|
||||
if handler is None:
|
||||
parser.print_help()
|
||||
|
||||
@@ -8,6 +8,8 @@ Stellt den KnowledgeStore, YAMLIndex, IndexEntry und das Artefakt-Modell bereit:
|
||||
- Artifact/ArtifactMetadata: Vollständige Wissensartefakte mit YAML-Frontmatter
|
||||
- ETLPipeline: ETL-Pipeline für inkrementelle Verarbeitung
|
||||
- MarkdownSource: Quellstrategie für Markdown-Dateien
|
||||
- IngestionPipeline: Neuer Orchestrator für den vollständigen Ingestion-Workflow
|
||||
- QuickCapture: Schnelle Wissenserfassung über CLI
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
@@ -21,19 +23,29 @@ from monorepo.knowledge.artifact import (
|
||||
)
|
||||
from monorepo.knowledge.etl import ETLPipeline, IngestError, IngestResult
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline, IngestionResult
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource, SourceError
|
||||
from monorepo.knowledge.store import KnowledgeStore, SearchResult
|
||||
from monorepo.knowledge.store import (
|
||||
ContextInjectionResult,
|
||||
KnowledgeStore,
|
||||
SearchResult,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Artifact",
|
||||
"ArtifactLink",
|
||||
"ArtifactMetadata",
|
||||
"ContextInjectionResult",
|
||||
"ETLPipeline",
|
||||
"IndexEntry",
|
||||
"IngestError",
|
||||
"IngestResult",
|
||||
"IngestionPipeline",
|
||||
"IngestionResult",
|
||||
"KnowledgeStore",
|
||||
"MarkdownSource",
|
||||
"QuickCapture",
|
||||
"SearchResult",
|
||||
"SourceError",
|
||||
"YAMLIndex",
|
||||
|
||||
@@ -39,6 +39,10 @@ class ArtifactMetadata:
|
||||
|
||||
Enthält mindestens: Typ, Titel, Tags, Quellkontext, Erstelldatum,
|
||||
Aktualisierungsdatum, Teilbarkeits-Flag, Verknüpfungen, Content-Hash.
|
||||
|
||||
Knowledge-Management-Erweiterungen (alle optional mit Defaults für
|
||||
Rückwärtskompatibilität): source, category, people, projects,
|
||||
enrichment_pending, ocr_failed, fetch_failed.
|
||||
"""
|
||||
|
||||
type: str
|
||||
@@ -51,6 +55,17 @@ class ArtifactMetadata:
|
||||
links: list[ArtifactLink] = field(default_factory=list)
|
||||
content_hash: str = ""
|
||||
|
||||
# Knowledge Management extensions
|
||||
source: dict[str, str] = field(default_factory=dict)
|
||||
# e.g. {"type": "confluence", "url": "...", "space": "...", "file": "..."}
|
||||
|
||||
category: str = "" # meeting, decision, project, reference, link, inbox
|
||||
people: list[str] = field(default_factory=list) # [[people/name]] refs
|
||||
projects: list[str] = field(default_factory=list) # [[projects/slug]] refs
|
||||
enrichment_pending: bool = False
|
||||
ocr_failed: bool = False
|
||||
fetch_failed: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Artifact:
|
||||
@@ -186,6 +201,21 @@ def parse_frontmatter(file_path: Path) -> Artifact:
|
||||
raw: dict[str, Any] = yaml.safe_load(yaml_text) or {}
|
||||
|
||||
# Metadaten extrahieren
|
||||
raw_source = raw.get("source")
|
||||
source: dict[str, str] = {}
|
||||
if isinstance(raw_source, dict):
|
||||
source = {str(k): str(v) for k, v in raw_source.items()}
|
||||
|
||||
raw_people = raw.get("people")
|
||||
people: list[str] = (
|
||||
[str(p) for p in raw_people] if isinstance(raw_people, list) else []
|
||||
)
|
||||
|
||||
raw_projects = raw.get("projects")
|
||||
projects: list[str] = (
|
||||
[str(p) for p in raw_projects] if isinstance(raw_projects, list) else []
|
||||
)
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type=str(raw.get("type", "")),
|
||||
title=str(raw.get("title", "")),
|
||||
@@ -196,6 +226,13 @@ def parse_frontmatter(file_path: Path) -> Artifact:
|
||||
shareable=bool(raw.get("shareable", False)),
|
||||
links=_parse_links(raw.get("links")),
|
||||
content_hash=str(raw.get("content_hash", "")),
|
||||
source=source,
|
||||
category=str(raw.get("category", "")),
|
||||
people=people,
|
||||
projects=projects,
|
||||
enrichment_pending=bool(raw.get("enrichment_pending", False)),
|
||||
ocr_failed=bool(raw.get("ocr_failed", False)),
|
||||
fetch_failed=bool(raw.get("fetch_failed", False)),
|
||||
)
|
||||
|
||||
return Artifact(metadata=metadata, content=content, file_path=file_path)
|
||||
@@ -236,6 +273,28 @@ def _serialize_metadata(metadata: ArtifactMetadata) -> dict[str, Any]:
|
||||
if metadata.content_hash:
|
||||
data["content_hash"] = metadata.content_hash
|
||||
|
||||
# Knowledge Management extensions (nur wenn nicht-leer/nicht-default)
|
||||
if metadata.source:
|
||||
data["source"] = metadata.source
|
||||
|
||||
if metadata.category:
|
||||
data["category"] = metadata.category
|
||||
|
||||
if metadata.people:
|
||||
data["people"] = metadata.people
|
||||
|
||||
if metadata.projects:
|
||||
data["projects"] = metadata.projects
|
||||
|
||||
if metadata.enrichment_pending:
|
||||
data["enrichment_pending"] = metadata.enrichment_pending
|
||||
|
||||
if metadata.ocr_failed:
|
||||
data["ocr_failed"] = metadata.ocr_failed
|
||||
|
||||
if metadata.fetch_failed:
|
||||
data["fetch_failed"] = metadata.fetch_failed
|
||||
|
||||
return data
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,728 @@
|
||||
"""CLI-Schnittstelle für das Knowledge Management System.
|
||||
|
||||
Stellt das `knowledge`-Subcommand des Monorepo-CLI bereit mit folgenden
|
||||
Unterkommandos:
|
||||
- capture: Quick-Capture von Text, URLs oder Dateipfaden
|
||||
- ingest: Pipeline-Durchlauf starten
|
||||
- search: Suche über den Knowledge-Index
|
||||
- status: Übersicht über Inbox-Elemente und letzte Verarbeitung
|
||||
- link: Web-Link als Referenzmaterial speichern
|
||||
|
||||
Requirements: 15.1, 15.2, 15.3, 15.4, 15.5, 15.6, 15.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_CONTEXTS = ("bahn", "dhive", "privat")
|
||||
|
||||
_HELP_TEXT = """\
|
||||
Knowledge Management – Wissen erfassen, verarbeiten und durchsuchen.
|
||||
|
||||
Unterkommandos:
|
||||
capture Schnelle Wissenserfassung (Text, URL oder Dateipfad)
|
||||
ingest Ingestion-Pipeline für konfigurierte Quellen starten
|
||||
search Knowledge-Index durchsuchen
|
||||
status Übersicht über Inbox und Artefakte
|
||||
link Web-Link als Referenzmaterial speichern
|
||||
|
||||
Beispiele:
|
||||
monorepo knowledge capture "Meeting-Notiz: API-Design besprochen" --context bahn
|
||||
monorepo knowledge capture https://docs.example.com/api --tags api,referenz
|
||||
monorepo knowledge ingest --context dhive --dry-run
|
||||
monorepo knowledge search "API Design" --limit 5
|
||||
monorepo knowledge status --context privat
|
||||
monorepo knowledge link https://example.com --title "Beispiel" --tags ref
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _find_monorepo_root() -> Path:
|
||||
"""Find the monorepo root directory.
|
||||
|
||||
Delegates to the main CLI's root-finding logic.
|
||||
|
||||
Returns:
|
||||
Absolute path to the monorepo root.
|
||||
"""
|
||||
from monorepo.cli import _find_monorepo_root as _main_find_root
|
||||
|
||||
return _main_find_root()
|
||||
|
||||
|
||||
def _derive_context_from_cwd() -> str | None:
|
||||
"""Derive context from the current working directory.
|
||||
|
||||
Checks if the CWD path contains a known context segment
|
||||
(/bahn/, /dhive/, /privat/).
|
||||
|
||||
Returns:
|
||||
The context string or None if it cannot be determined.
|
||||
"""
|
||||
cwd = Path.cwd()
|
||||
path_str = str(cwd).replace("\\", "/")
|
||||
|
||||
for ctx in VALID_CONTEXTS:
|
||||
if f"/{ctx}/" in path_str or path_str.endswith(f"/{ctx}"):
|
||||
return ctx
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_context(context_flag: str | None) -> str:
|
||||
"""Resolve context from --context flag or current working directory.
|
||||
|
||||
Args:
|
||||
context_flag: Explicit context from CLI flag, or None.
|
||||
|
||||
Returns:
|
||||
The resolved context string.
|
||||
|
||||
Raises:
|
||||
SystemExit: If context cannot be determined.
|
||||
"""
|
||||
if context_flag:
|
||||
if context_flag not in VALID_CONTEXTS:
|
||||
print(
|
||||
f"Fehler: Ungültiger Kontext '{context_flag}'. "
|
||||
f"Erlaubt: {', '.join(VALID_CONTEXTS)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
return context_flag
|
||||
|
||||
derived = _derive_context_from_cwd()
|
||||
if derived:
|
||||
return derived
|
||||
|
||||
print(
|
||||
"Fehler: Kontext konnte nicht aus dem Arbeitsverzeichnis abgeleitet werden.\n"
|
||||
"Verwende --context <bahn|dhive|privat> um den Kontext explizit anzugeben.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parser Builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_knowledge_parser() -> argparse.ArgumentParser:
|
||||
"""Build a standalone argument parser for the knowledge subcommand.
|
||||
|
||||
This can be used independently for testing or as part of the
|
||||
main CLI parser registration.
|
||||
|
||||
Returns:
|
||||
ArgumentParser configured with all knowledge subcommands.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="knowledge",
|
||||
description=_HELP_TEXT,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(
|
||||
dest="knowledge_command",
|
||||
help="Knowledge-Unterkommandos",
|
||||
)
|
||||
|
||||
# --- knowledge capture ---
|
||||
p_capture = subparsers.add_parser(
|
||||
"capture",
|
||||
help="Text, URL oder Dateipfad schnell erfassen",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"input",
|
||||
help="Zu erfassender Text, URL oder Dateipfad.",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"--tags", "-t",
|
||||
default=None,
|
||||
help="Komma-getrennte Tags (z.B. 'api,meeting').",
|
||||
)
|
||||
|
||||
# --- knowledge ingest ---
|
||||
p_ingest = subparsers.add_parser(
|
||||
"ingest",
|
||||
help="Ingestion-Pipeline für konfigurierte Quellen starten",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Vorschau ohne Schreiboperationen.",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--verbose", "-v",
|
||||
action="store_true",
|
||||
help="Detaillierte Ausgabe.",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--no-commit",
|
||||
action="store_true",
|
||||
help="Git-Commit überspringen.",
|
||||
)
|
||||
|
||||
# --- knowledge search ---
|
||||
p_search = subparsers.add_parser(
|
||||
"search",
|
||||
help="Knowledge-Index durchsuchen",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"query",
|
||||
help="Suchbegriff.",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Suche auf einen Kontext beschränken.",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--limit", "-l",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Maximale Anzahl Ergebnisse (Standard: 20).",
|
||||
)
|
||||
|
||||
# --- knowledge status ---
|
||||
p_status = subparsers.add_parser(
|
||||
"status",
|
||||
help="Übersicht über Inbox-Elemente und Artefakte",
|
||||
)
|
||||
p_status.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Kontext anzeigen (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
|
||||
# --- knowledge link ---
|
||||
p_link = subparsers.add_parser(
|
||||
"link",
|
||||
help="Web-Link als Referenzmaterial speichern",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"url",
|
||||
help="Die zu speichernde URL.",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--title", "-T",
|
||||
default=None,
|
||||
help="Optionaler Titel für den Link.",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--tags", "-t",
|
||||
default=None,
|
||||
help="Komma-getrennte Tags (z.B. 'api,referenz').",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand Handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cmd_capture(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge capture <text|url|path> [--context CTX] [--tags TAG,TAG]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with input, context, tags.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for failure).
|
||||
"""
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
|
||||
root = _find_monorepo_root()
|
||||
context = _resolve_context(args.context)
|
||||
|
||||
# Parse tags
|
||||
tags: list[str] = []
|
||||
if args.tags:
|
||||
tags = [t.strip() for t in args.tags.split(",") if t.strip()]
|
||||
|
||||
capture = QuickCapture(monorepo_root=root)
|
||||
try:
|
||||
result_path = capture.capture(
|
||||
input_text=args.input,
|
||||
context=context,
|
||||
tags=tags,
|
||||
)
|
||||
print(f"✓ Erfasst: {result_path}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"Fehler beim Erfassen: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_ingest(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge ingest [--context CTX] [--dry-run] [--verbose] [--no-commit]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with context, dry_run, verbose, no_commit.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for failure).
|
||||
"""
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
|
||||
root = _find_monorepo_root()
|
||||
context = _resolve_context(args.context)
|
||||
|
||||
# Configure logging for verbose mode
|
||||
if args.verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(levelname)s: %(name)s: %(message)s",
|
||||
force=True,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=root,
|
||||
context=context,
|
||||
dry_run=args.dry_run,
|
||||
no_commit=args.no_commit,
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
print(f"[DRY RUN] Ingestion für Kontext '{context}'...")
|
||||
else:
|
||||
print(f"Ingestion für Kontext '{context}'...")
|
||||
|
||||
try:
|
||||
result = pipeline.run()
|
||||
except Exception as e:
|
||||
print(f"Fehler bei Ingestion: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Print results
|
||||
print(f"\n{'─' * 40}")
|
||||
print(f"Kontext: {result.context}")
|
||||
print(f"Verarbeitet: {result.processed}")
|
||||
print(f"Übersprungen: {result.skipped}")
|
||||
if hasattr(result, "tasks_created") and result.tasks_created:
|
||||
print(f"Tasks erstellt: {result.tasks_created}")
|
||||
if hasattr(result, "commit_sha") and result.commit_sha:
|
||||
print(f"Commit: {result.commit_sha[:8]}")
|
||||
|
||||
if result.errors:
|
||||
print(f"\n⚠ {len(result.errors)} Fehler:")
|
||||
for err in result.errors:
|
||||
print(f" • [{err.source}] {err.message}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_search(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge search <query> [--context CTX] [--limit N]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with query, context, limit.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for failure).
|
||||
"""
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
root = _find_monorepo_root()
|
||||
|
||||
# Determine allowed scopes based on context flag
|
||||
if args.context:
|
||||
if args.context not in VALID_CONTEXTS:
|
||||
print(
|
||||
f"Fehler: Ungültiger Kontext '{args.context}'.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
allowed_scopes = [args.context]
|
||||
else:
|
||||
allowed_scopes = list(VALID_CONTEXTS) + ["shared"]
|
||||
|
||||
# Create KnowledgeStore
|
||||
ks_path = root / "shared" / "knowledge-store"
|
||||
scope_config = ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
})
|
||||
store = KnowledgeStore(base_path=ks_path, scope_config=scope_config)
|
||||
|
||||
query = args.query
|
||||
results = store.search_cross_context(query=query, allowed_scopes=allowed_scopes)
|
||||
|
||||
# Apply limit
|
||||
limit = args.limit or 20
|
||||
results = results[:limit]
|
||||
|
||||
if not results:
|
||||
print(f"Keine Treffer für '{query}'.")
|
||||
return 0
|
||||
|
||||
print(f"Treffer für '{query}' ({len(results)} Ergebnis(se)):\n")
|
||||
for r in results:
|
||||
scope_label = r.entry.scope if r.entry.scope else "?"
|
||||
tags_str = ", ".join(r.entry.tags) if r.entry.tags else "-"
|
||||
entry_type = getattr(r.entry, "type", None) or "-"
|
||||
print(f" [{scope_label}] {r.entry.title}")
|
||||
print(f" Typ: {entry_type}")
|
||||
print(f" Tags: {tags_str}")
|
||||
print(f" Pfad: {r.entry.path}")
|
||||
print()
|
||||
|
||||
if results and results[0].partial_results:
|
||||
print("⚠ Suche wegen Timeout abgebrochen – möglicherweise unvollständig.")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_status(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge status [--context CTX]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with context.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success).
|
||||
"""
|
||||
root = _find_monorepo_root()
|
||||
context = _resolve_context(args.context)
|
||||
|
||||
knowledge_path = root / context / "knowledge"
|
||||
inbox_path = knowledge_path / "inbox"
|
||||
|
||||
# Count inbox items
|
||||
inbox_items: list[Path] = []
|
||||
if inbox_path.exists():
|
||||
inbox_items = [
|
||||
f for f in inbox_path.iterdir()
|
||||
if f.is_file() and f.suffix == ".md" and f.name != "index.md"
|
||||
]
|
||||
|
||||
# Count total artifacts across all subfolders
|
||||
total_artifacts = 0
|
||||
subfolders = ["meetings", "decisions", "projects", "people", "references", "links"]
|
||||
folder_counts: dict[str, int] = {}
|
||||
|
||||
for folder_name in subfolders:
|
||||
folder_path = knowledge_path / folder_name
|
||||
if folder_path.exists():
|
||||
count = sum(
|
||||
1 for f in folder_path.iterdir()
|
||||
if f.is_file() and f.suffix == ".md" and f.name != "index.md"
|
||||
)
|
||||
folder_counts[folder_name] = count
|
||||
total_artifacts += count
|
||||
|
||||
# Check _index.yaml
|
||||
index_path = knowledge_path / "_index.yaml"
|
||||
index_exists = index_path.exists()
|
||||
|
||||
# Check sources.yaml
|
||||
sources_path = knowledge_path / "sources.yaml"
|
||||
sources_exists = sources_path.exists()
|
||||
|
||||
# Output
|
||||
print(f"Knowledge Status: {context}")
|
||||
print(f"{'─' * 40}")
|
||||
print(f"Pfad: {knowledge_path}")
|
||||
print(f"Index: {'✓' if index_exists else '✗'}")
|
||||
print(f"Sources: {'✓' if sources_exists else '✗'}")
|
||||
print(f"\nInbox: {len(inbox_items)} Element(e)")
|
||||
print(f"Artefakte: {total_artifacts} gesamt")
|
||||
|
||||
if folder_counts:
|
||||
print()
|
||||
for folder, count in sorted(folder_counts.items()):
|
||||
if count > 0:
|
||||
print(f" {folder:12s} {count}")
|
||||
|
||||
if inbox_items:
|
||||
print(f"\nInbox-Elemente:")
|
||||
for item in sorted(inbox_items, key=lambda p: p.name, reverse=True)[:10]:
|
||||
print(f" • {item.name}")
|
||||
if len(inbox_items) > 10:
|
||||
print(f" ... und {len(inbox_items) - 10} weitere")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_link(args: argparse.Namespace) -> int:
|
||||
"""Handle `knowledge link <url> [--title TITLE] [--tags TAG,TAG] [--context CTX]`.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments with url, title, tags, context.
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for failure).
|
||||
"""
|
||||
from monorepo.knowledge.sources.link import LinkRegistry
|
||||
|
||||
root = _find_monorepo_root()
|
||||
context = _resolve_context(args.context)
|
||||
|
||||
# Parse tags
|
||||
tags: list[str] = []
|
||||
if args.tags:
|
||||
tags = [t.strip() for t in args.tags.split(",") if t.strip()]
|
||||
|
||||
context_path = root / context / "knowledge"
|
||||
registry = LinkRegistry(context_path)
|
||||
|
||||
try:
|
||||
result_path = registry.save_link(
|
||||
url=args.url,
|
||||
title=args.title,
|
||||
tags=tags,
|
||||
)
|
||||
print(f"✓ Link gespeichert: {result_path}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"Fehler beim Speichern des Links: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main Entry Point (standalone usage)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def knowledge_main(args: argparse.Namespace) -> None:
|
||||
"""Main entry point for standalone knowledge CLI usage.
|
||||
|
||||
Dispatches to the appropriate handler based on knowledge_command.
|
||||
Calls sys.exit when no command is given.
|
||||
|
||||
Args:
|
||||
args: Parsed argparse namespace.
|
||||
"""
|
||||
knowledge_cmd = getattr(args, "knowledge_command", None)
|
||||
|
||||
if not knowledge_cmd:
|
||||
print(_HELP_TEXT)
|
||||
sys.exit(0)
|
||||
|
||||
dispatch = {
|
||||
"capture": _cmd_capture,
|
||||
"ingest": _cmd_ingest,
|
||||
"search": _cmd_search,
|
||||
"status": _cmd_status,
|
||||
"link": _cmd_link,
|
||||
}
|
||||
|
||||
handler = dispatch.get(knowledge_cmd)
|
||||
if handler is None:
|
||||
print(f"Unbekanntes Unterkommando: {knowledge_cmd}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
exit_code = handler(args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration for main CLI integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register_knowledge_subcommand(subparsers: argparse._SubParsersAction) -> None:
|
||||
"""Register the `knowledge` subcommand group under the main CLI parser.
|
||||
|
||||
This reuses the same parser structure as build_knowledge_parser but
|
||||
integrates it as a subparser of the main monorepo CLI.
|
||||
|
||||
Args:
|
||||
subparsers: The subparsers action from the main argparse parser.
|
||||
"""
|
||||
knowledge_parser = subparsers.add_parser(
|
||||
"knowledge",
|
||||
help="Wissensmanagement: Erfassen, Verarbeiten, Suchen",
|
||||
description=_HELP_TEXT,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
# Sub-subparsers for knowledge commands
|
||||
knowledge_sub = knowledge_parser.add_subparsers(
|
||||
dest="knowledge_command",
|
||||
help="Knowledge-Unterkommandos",
|
||||
)
|
||||
|
||||
# --- knowledge capture ---
|
||||
p_capture = knowledge_sub.add_parser(
|
||||
"capture",
|
||||
help="Text, URL oder Dateipfad schnell erfassen",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"input",
|
||||
help="Zu erfassender Text, URL oder Dateipfad.",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
p_capture.add_argument(
|
||||
"--tags", "-t",
|
||||
default=None,
|
||||
help="Komma-getrennte Tags (z.B. 'api,meeting').",
|
||||
)
|
||||
|
||||
# --- knowledge ingest ---
|
||||
p_ingest = knowledge_sub.add_parser(
|
||||
"ingest",
|
||||
help="Ingestion-Pipeline für konfigurierte Quellen starten",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Vorschau ohne Schreiboperationen.",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--verbose", "-v",
|
||||
action="store_true",
|
||||
help="Detaillierte Ausgabe.",
|
||||
)
|
||||
p_ingest.add_argument(
|
||||
"--no-commit",
|
||||
action="store_true",
|
||||
help="Git-Commit überspringen.",
|
||||
)
|
||||
|
||||
# --- knowledge search ---
|
||||
p_search = knowledge_sub.add_parser(
|
||||
"search",
|
||||
help="Knowledge-Index durchsuchen",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"query",
|
||||
help="Suchbegriff.",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Suche auf einen Kontext beschränken.",
|
||||
)
|
||||
p_search.add_argument(
|
||||
"--limit", "-l",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Maximale Anzahl Ergebnisse (Standard: 20).",
|
||||
)
|
||||
|
||||
# --- knowledge status ---
|
||||
p_status = knowledge_sub.add_parser(
|
||||
"status",
|
||||
help="Übersicht über Inbox-Elemente und Artefakte",
|
||||
)
|
||||
p_status.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Kontext anzeigen (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
|
||||
# --- knowledge link ---
|
||||
p_link = knowledge_sub.add_parser(
|
||||
"link",
|
||||
help="Web-Link als Referenzmaterial speichern",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"url",
|
||||
help="Die zu speichernde URL.",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--title", "-T",
|
||||
default=None,
|
||||
help="Optionaler Titel für den Link.",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--tags", "-t",
|
||||
default=None,
|
||||
help="Komma-getrennte Tags (z.B. 'api,referenz').",
|
||||
)
|
||||
p_link.add_argument(
|
||||
"--context", "-c",
|
||||
choices=VALID_CONTEXTS,
|
||||
default=None,
|
||||
help="Zielkontext (Standard: aus CWD abgeleitet).",
|
||||
)
|
||||
|
||||
|
||||
def handle_knowledge_command(args: argparse.Namespace, root: Path) -> int:
|
||||
"""Dispatch knowledge subcommands from the main CLI.
|
||||
|
||||
Called from the main CLI's main() when 'knowledge' is the active command.
|
||||
Uses the root provided by the main CLI instead of discovering it again.
|
||||
|
||||
Args:
|
||||
args: Parsed arguments namespace.
|
||||
root: Monorepo root path (provided by main CLI).
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, non-zero for failure).
|
||||
"""
|
||||
knowledge_cmd = getattr(args, "knowledge_command", None)
|
||||
|
||||
if not knowledge_cmd:
|
||||
print(_HELP_TEXT)
|
||||
return 0
|
||||
|
||||
dispatch = {
|
||||
"capture": _cmd_capture,
|
||||
"ingest": _cmd_ingest,
|
||||
"search": _cmd_search,
|
||||
"status": _cmd_status,
|
||||
"link": _cmd_link,
|
||||
}
|
||||
|
||||
handler = dispatch.get(knowledge_cmd)
|
||||
if handler is None:
|
||||
print(f"Unbekanntes Unterkommando: {knowledge_cmd}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
return handler(args)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Unified Ingestion Pipeline für den Wissensspeicher.
|
||||
|
||||
Orchestriert das Erfassen, Routen, Anreichern und Speichern von
|
||||
Wissensartefakten aus verschiedenen Quellen in die Knowledge-Kontexte.
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
from monorepo.knowledge.ingestion.config import (
|
||||
ConfigValidationError,
|
||||
EnrichmentSettings,
|
||||
GitSettings,
|
||||
OCRSettings,
|
||||
PipelineSettings,
|
||||
SourceConfig,
|
||||
load_sources_config,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.enrichment import (
|
||||
ActionItem,
|
||||
EnrichmentAgent,
|
||||
EnrichmentResult,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.git_integration import (
|
||||
auto_commit,
|
||||
auto_commit_capture,
|
||||
generate_capture_commit_message,
|
||||
generate_commit_message,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.pipeline import (
|
||||
IngestError,
|
||||
IngestionPipeline,
|
||||
IngestionResult,
|
||||
get_source_strategy,
|
||||
register_source_strategy,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.router import ContextRouter, RoutingDecision
|
||||
|
||||
__all__ = [
|
||||
"ActionItem",
|
||||
"ConfigValidationError",
|
||||
"ContextRouter",
|
||||
"EnrichmentAgent",
|
||||
"EnrichmentResult",
|
||||
"EnrichmentSettings",
|
||||
"GitSettings",
|
||||
"IngestError",
|
||||
"IngestionPipeline",
|
||||
"IngestionResult",
|
||||
"OCRSettings",
|
||||
"PipelineSettings",
|
||||
"QuickCapture",
|
||||
"RoutingDecision",
|
||||
"SourceConfig",
|
||||
"auto_commit",
|
||||
"auto_commit_capture",
|
||||
"generate_capture_commit_message",
|
||||
"generate_commit_message",
|
||||
"get_source_strategy",
|
||||
"load_sources_config",
|
||||
"register_source_strategy",
|
||||
]
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Capture – Schnelle Wissenserfassung über CLI und Integrationen.
|
||||
|
||||
Ermöglicht das ad-hoc Erfassen von Wissensschnipseln, Notizen und
|
||||
Erkenntnissen ohne vollständigen Pipeline-Durchlauf.
|
||||
|
||||
Requirements: 3.1, 3.2, 3.3, 3.7, 8.3
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from monorepo.knowledge.ingestion.router import ContextRouter
|
||||
|
||||
|
||||
class QuickCapture:
|
||||
"""Fast knowledge capture to inbox."""
|
||||
|
||||
def __init__(self, monorepo_root: Path) -> None:
|
||||
"""Initialize with monorepo root path.
|
||||
|
||||
Args:
|
||||
monorepo_root: Absolute path to the monorepo root directory.
|
||||
"""
|
||||
self.monorepo_root = monorepo_root
|
||||
self._router = ContextRouter(monorepo_root)
|
||||
|
||||
def capture(
|
||||
self,
|
||||
input_text: str,
|
||||
context: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
) -> Path:
|
||||
"""Capture text/URL/filepath to inbox. Returns path of created artifact.
|
||||
|
||||
Args:
|
||||
input_text: The text, URL, or file path to capture.
|
||||
context: Target context (bahn, dhive, privat). If None,
|
||||
derived from current working directory.
|
||||
tags: Optional list of tags.
|
||||
|
||||
Returns:
|
||||
Path of the created Markdown artifact.
|
||||
|
||||
Raises:
|
||||
ValueError: If context cannot be determined.
|
||||
"""
|
||||
if tags is None:
|
||||
tags = []
|
||||
|
||||
# Determine context
|
||||
if context is None:
|
||||
context = self._router.determine_context(cwd=Path.cwd())
|
||||
|
||||
# Detect input type
|
||||
input_type = self._detect_input_type(input_text)
|
||||
|
||||
# Generate title based on input type
|
||||
title = self._derive_title(input_text, input_type)
|
||||
|
||||
# Generate filename
|
||||
filename = self._generate_filename(title)
|
||||
|
||||
# Build target path: {context}/knowledge/inbox/{filename}
|
||||
target_dir = self.monorepo_root / context / "knowledge" / "inbox"
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
target_path = target_dir / filename
|
||||
|
||||
# Build frontmatter
|
||||
now = datetime.now()
|
||||
frontmatter = self._build_frontmatter(
|
||||
title=title,
|
||||
created=now.strftime("%Y-%m-%d"),
|
||||
source_type=input_type,
|
||||
source_value=input_text,
|
||||
tags=tags,
|
||||
source_context=context,
|
||||
)
|
||||
|
||||
# Build content body
|
||||
body = self._build_body(input_text, input_type)
|
||||
|
||||
# Write file
|
||||
content = f"{frontmatter}\n{body}\n"
|
||||
target_path.write_text(content, encoding="utf-8")
|
||||
|
||||
return target_path
|
||||
|
||||
def _detect_input_type(self, input_text: str) -> str:
|
||||
"""Detect if input is URL, file path, or plain text.
|
||||
|
||||
Args:
|
||||
input_text: The raw input string.
|
||||
|
||||
Returns:
|
||||
One of 'url', 'file', or 'text'.
|
||||
"""
|
||||
stripped = input_text.strip()
|
||||
|
||||
# Check for URL
|
||||
if stripped.startswith("http://") or stripped.startswith("https://"):
|
||||
return "url"
|
||||
|
||||
# Check for file path
|
||||
if self._looks_like_filepath(stripped):
|
||||
return "file"
|
||||
|
||||
return "text"
|
||||
|
||||
def _looks_like_filepath(self, text: str) -> bool:
|
||||
"""Heuristic: does the text look like a file path?
|
||||
|
||||
Returns True if:
|
||||
- It contains path separators AND has a file extension, OR
|
||||
- Path(text).exists()
|
||||
"""
|
||||
# Contains separators and has an extension
|
||||
has_separator = "/" in text or "\\" in text
|
||||
has_extension = bool(re.search(r"\.\w{1,10}$", text))
|
||||
|
||||
if has_separator and has_extension:
|
||||
return True
|
||||
|
||||
# Actually exists on disk
|
||||
try:
|
||||
return Path(text).exists()
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
def _derive_title(self, input_text: str, input_type: str) -> str:
|
||||
"""Derive a title from the input based on its type.
|
||||
|
||||
Args:
|
||||
input_text: The raw input.
|
||||
input_type: One of 'url', 'file', 'text'.
|
||||
|
||||
Returns:
|
||||
A human-readable title string.
|
||||
"""
|
||||
stripped = input_text.strip()
|
||||
|
||||
if input_type == "url":
|
||||
parsed = urlparse(stripped)
|
||||
# domain + path, strip leading/trailing slashes
|
||||
path_part = parsed.path.strip("/")
|
||||
if path_part:
|
||||
return f"{parsed.netloc}/{path_part}"
|
||||
return parsed.netloc
|
||||
|
||||
if input_type == "file":
|
||||
return Path(stripped).stem
|
||||
|
||||
# Text: first 60 characters
|
||||
first_line = stripped.split("\n", 1)[0]
|
||||
return first_line[:60]
|
||||
|
||||
def _generate_filename(self, title: str) -> str:
|
||||
"""Generate YYYY-MM-DD-HH-MM-slugified-title.md filename.
|
||||
|
||||
Args:
|
||||
title: The title to slugify for the filename.
|
||||
|
||||
Returns:
|
||||
A filename string with timestamp prefix and slugified title.
|
||||
"""
|
||||
now = datetime.now()
|
||||
timestamp = now.strftime("%Y-%m-%d-%H-%M")
|
||||
|
||||
slug = self._slugify(title)
|
||||
# Limit slug to 50 chars
|
||||
slug = slug[:50].rstrip("-")
|
||||
|
||||
# Fallback if slugification removed all characters (e.g. non-ASCII-only titles)
|
||||
if not slug:
|
||||
slug = "untitled"
|
||||
|
||||
return f"{timestamp}-{slug}.md"
|
||||
|
||||
def _slugify(self, text: str) -> str:
|
||||
"""Convert text to URL-safe slug.
|
||||
|
||||
Lowercase, replace non-alphanumeric with hyphens,
|
||||
collapse multiple hyphens, strip leading/trailing hyphens.
|
||||
"""
|
||||
slug = text.lower()
|
||||
# Replace non-alphanumeric (keep hyphens) with hyphens
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
||||
# Collapse multiple hyphens
|
||||
slug = re.sub(r"-{2,}", "-", slug)
|
||||
# Strip leading/trailing hyphens
|
||||
slug = slug.strip("-")
|
||||
return slug
|
||||
|
||||
def _build_frontmatter(
|
||||
self,
|
||||
title: str,
|
||||
created: str,
|
||||
source_type: str,
|
||||
source_value: str,
|
||||
tags: list[str],
|
||||
source_context: str,
|
||||
) -> str:
|
||||
"""Build YAML frontmatter block.
|
||||
|
||||
Args:
|
||||
title: Artifact title.
|
||||
created: Creation date (YYYY-MM-DD).
|
||||
source_type: Type of source (url, file, text).
|
||||
source_value: Original input value.
|
||||
tags: List of tags.
|
||||
source_context: The target context.
|
||||
|
||||
Returns:
|
||||
Complete YAML frontmatter as string including delimiters.
|
||||
"""
|
||||
# Escape title for YAML (wrap in quotes if it contains special chars)
|
||||
escaped_title = title.replace('"', '\\"')
|
||||
|
||||
lines = [
|
||||
"---",
|
||||
"type: inbox",
|
||||
f'title: "{escaped_title}"',
|
||||
f"created: {created}",
|
||||
"source:",
|
||||
f" type: {source_type}",
|
||||
f' value: "{source_value}"',
|
||||
]
|
||||
|
||||
if tags:
|
||||
tags_str = ", ".join(tags)
|
||||
lines.append(f"tags: [{tags_str}]")
|
||||
else:
|
||||
lines.append("tags: []")
|
||||
|
||||
lines.append(f"source_context: {source_context}")
|
||||
lines.append("---")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _build_body(self, input_text: str, input_type: str) -> str:
|
||||
"""Build the Markdown content body.
|
||||
|
||||
Args:
|
||||
input_text: The raw input.
|
||||
input_type: One of 'url', 'file', 'text'.
|
||||
|
||||
Returns:
|
||||
Markdown body content.
|
||||
"""
|
||||
stripped = input_text.strip()
|
||||
|
||||
if input_type == "url":
|
||||
return f"\n[{stripped}]({stripped})\n"
|
||||
|
||||
if input_type == "file":
|
||||
return f"\nSource file: `{stripped}`\n"
|
||||
|
||||
# Plain text
|
||||
return f"\n{stripped}\n"
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Config – Konfigurationsmodell für die Ingestion-Pipeline.
|
||||
|
||||
Definiert das Schema für sources.yaml und Pipeline-Einstellungen
|
||||
pro Kontext. Unterstützt Umgebungsvariablen-Auflösung (${VAR_NAME})
|
||||
und Validierung der Konfiguration vor Pipeline-Start.
|
||||
|
||||
Requirements: 12.1, 12.2, 12.4, 12.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ConfigValidationError(Exception):
|
||||
"""Wird ausgelöst bei ungültiger sources.yaml-Konfiguration.
|
||||
|
||||
Enthält eine Liste aller gefundenen Validierungsfehler.
|
||||
"""
|
||||
|
||||
def __init__(self, errors: list[str]) -> None:
|
||||
self.errors = errors
|
||||
msg = "Konfigurationsvalidierung fehlgeschlagen:\n" + "\n".join(
|
||||
f" - {e}" for e in errors
|
||||
)
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Datenmodelle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceConfig:
|
||||
"""Configuration for a single source in sources.yaml."""
|
||||
|
||||
type: str # confluence, jira, email, file, link, wissensdatenbank
|
||||
name: str # human-readable identifier
|
||||
params: dict[str, Any] = field(default_factory=dict) # type-specific params
|
||||
target_folder: str = "" # override target subfolder
|
||||
sync_frequency: str = "manual" # on-push, hourly, daily, manual
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichmentSettings:
|
||||
"""Einstellungen für den Enrichment-Agenten."""
|
||||
|
||||
provider: str = "kiro"
|
||||
model: str = ""
|
||||
confidence_threshold: float = 0.7
|
||||
|
||||
|
||||
@dataclass
|
||||
class OCRSettings:
|
||||
"""Einstellungen für die OCR-Engine."""
|
||||
|
||||
provider: str = "kiro"
|
||||
languages: str = "deu+eng"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GitSettings:
|
||||
"""Einstellungen für die Git-Integration."""
|
||||
|
||||
auto_commit: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineSettings:
|
||||
"""Pipeline-Einstellungen aus dem settings-Block in sources.yaml."""
|
||||
|
||||
create_tasks: bool = True
|
||||
enrichment: EnrichmentSettings = field(default_factory=EnrichmentSettings)
|
||||
ocr: OCRSettings = field(default_factory=OCRSettings)
|
||||
git: GitSettings = field(default_factory=GitSettings)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Umgebungsvariablen-Auflösung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_ENV_VAR_PATTERN = re.compile(r"\$\{([^}]+)\}")
|
||||
|
||||
|
||||
def resolve_env_vars(value: str) -> str:
|
||||
"""Ersetzt alle ${VAR_NAME}-Patterns in einem String durch Umgebungsvariablen.
|
||||
|
||||
Args:
|
||||
value: Der Eingabestring mit möglichen ${VAR_NAME}-Patterns.
|
||||
|
||||
Returns:
|
||||
Der String mit aufgelösten Umgebungsvariablen.
|
||||
|
||||
Raises:
|
||||
ConfigValidationError: Wenn eine referenzierte Variable nicht gesetzt ist.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
def _replace(match: re.Match[str]) -> str:
|
||||
var_name = match.group(1)
|
||||
env_value = os.environ.get(var_name)
|
||||
if env_value is None:
|
||||
errors.append(
|
||||
f"Umgebungsvariable '${{{var_name}}}' ist nicht gesetzt"
|
||||
)
|
||||
return match.group(0) # Originaltext beibehalten für Fehlermeldung
|
||||
return env_value
|
||||
|
||||
result = _ENV_VAR_PATTERN.sub(_replace, value)
|
||||
|
||||
if errors:
|
||||
raise ConfigValidationError(errors)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_env_vars_recursive(obj: Any) -> Any:
|
||||
"""Löst ${VAR_NAME}-Patterns rekursiv in verschachtelten Datenstrukturen auf.
|
||||
|
||||
Traversiert dicts, Listen und Strings. Nicht-String-Werte werden
|
||||
unverändert zurückgegeben.
|
||||
|
||||
Args:
|
||||
obj: Die zu traversierende Datenstruktur.
|
||||
|
||||
Returns:
|
||||
Die Datenstruktur mit aufgelösten Umgebungsvariablen.
|
||||
|
||||
Raises:
|
||||
ConfigValidationError: Wenn Variablen nicht aufgelöst werden können.
|
||||
"""
|
||||
if isinstance(obj, str):
|
||||
return resolve_env_vars(obj)
|
||||
elif isinstance(obj, dict):
|
||||
return {k: _resolve_env_vars_recursive(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, list):
|
||||
return [_resolve_env_vars_recursive(item) for item in obj]
|
||||
return obj
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validierung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _validate_source_entry(entry: dict[str, Any], index: int) -> list[str]:
|
||||
"""Validiert einen einzelnen Source-Eintrag.
|
||||
|
||||
Args:
|
||||
entry: Der rohe dict-Eintrag aus der YAML-Datei.
|
||||
index: Der Index in der sources-Liste (für Fehlermeldungen).
|
||||
|
||||
Returns:
|
||||
Liste von Validierungsfehlern (leer wenn gültig).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
if not isinstance(entry, dict):
|
||||
errors.append(f"Source #{index + 1}: Eintrag muss ein Mapping sein")
|
||||
return errors
|
||||
|
||||
# Pflichtfelder prüfen
|
||||
if not entry.get("type"):
|
||||
errors.append(f"Source #{index + 1}: Pflichtfeld 'type' fehlt oder ist leer")
|
||||
if not entry.get("name"):
|
||||
errors.append(f"Source #{index + 1}: Pflichtfeld 'name' fehlt oder ist leer")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _collect_env_var_errors(obj: Any, path: str = "") -> list[str]:
|
||||
"""Sammelt alle nicht auflösbaren Umgebungsvariablen ohne Exception.
|
||||
|
||||
Traversiert die Datenstruktur und identifiziert alle ${VAR_NAME}-Patterns,
|
||||
deren Variablen nicht gesetzt sind.
|
||||
|
||||
Args:
|
||||
obj: Die zu prüfende Datenstruktur.
|
||||
path: Der aktuelle Pfad für Fehlermeldungen.
|
||||
|
||||
Returns:
|
||||
Liste von Fehlermeldungen für nicht-auflösbare Variablen.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
if isinstance(obj, str):
|
||||
for match in _ENV_VAR_PATTERN.finditer(obj):
|
||||
var_name = match.group(1)
|
||||
if os.environ.get(var_name) is None:
|
||||
location = f" (in {path})" if path else ""
|
||||
errors.append(
|
||||
f"Umgebungsvariable '${{{var_name}}}' ist nicht gesetzt{location}"
|
||||
)
|
||||
elif isinstance(obj, dict):
|
||||
for key, value in obj.items():
|
||||
sub_path = f"{path}.{key}" if path else key
|
||||
errors.extend(_collect_env_var_errors(value, sub_path))
|
||||
elif isinstance(obj, list):
|
||||
for i, item in enumerate(obj):
|
||||
sub_path = f"{path}[{i}]"
|
||||
errors.extend(_collect_env_var_errors(item, sub_path))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sources.yaml Laden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_sources_config(path: Path) -> tuple[list[SourceConfig], PipelineSettings]:
|
||||
"""Parst eine sources.yaml-Datei und gibt Quellkonfigurationen + Settings zurück.
|
||||
|
||||
Ablauf:
|
||||
1. YAML-Datei einlesen
|
||||
2. Pflichtfelder validieren
|
||||
3. Umgebungsvariablen in allen String-Werten auflösen
|
||||
4. SourceConfig- und PipelineSettings-Objekte erstellen
|
||||
|
||||
Args:
|
||||
path: Pfad zur sources.yaml-Datei.
|
||||
|
||||
Returns:
|
||||
Tuple aus (Liste von SourceConfig, PipelineSettings).
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn die Datei nicht existiert.
|
||||
ConfigValidationError: Bei Validierungsfehlern (fehlende Felder,
|
||||
nicht-auflösbare Umgebungsvariablen).
|
||||
"""
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Konfigurationsdatei nicht gefunden: {path}")
|
||||
|
||||
text = path.read_text(encoding="utf-8")
|
||||
raw: dict[str, Any] = yaml.safe_load(text) or {}
|
||||
|
||||
# --- Phase 1: Strukturvalidierung ---
|
||||
validation_errors: list[str] = []
|
||||
|
||||
sources_raw = raw.get("sources", [])
|
||||
if not isinstance(sources_raw, list):
|
||||
validation_errors.append("'sources' muss eine Liste sein")
|
||||
sources_raw = []
|
||||
|
||||
for i, entry in enumerate(sources_raw):
|
||||
validation_errors.extend(_validate_source_entry(entry, i))
|
||||
|
||||
# --- Phase 2: Env-Var-Validierung (alle sammeln, nicht beim ersten abbrechen) ---
|
||||
for i, entry in enumerate(sources_raw):
|
||||
if isinstance(entry, dict):
|
||||
env_errors = _collect_env_var_errors(
|
||||
entry.get("params", {}), f"sources[{i}].params"
|
||||
)
|
||||
validation_errors.extend(env_errors)
|
||||
|
||||
# Settings-Block auch auf env vars prüfen
|
||||
settings_raw = raw.get("settings", {})
|
||||
if isinstance(settings_raw, dict):
|
||||
env_errors = _collect_env_var_errors(settings_raw, "settings")
|
||||
validation_errors.extend(env_errors)
|
||||
|
||||
if validation_errors:
|
||||
raise ConfigValidationError(validation_errors)
|
||||
|
||||
# --- Phase 3: Env-Vars auflösen und Objekte erstellen ---
|
||||
sources: list[SourceConfig] = []
|
||||
for entry in sources_raw:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
|
||||
# params mit aufgelösten env vars
|
||||
resolved_params = _resolve_env_vars_recursive(entry.get("params", {}))
|
||||
|
||||
sources.append(
|
||||
SourceConfig(
|
||||
type=str(entry.get("type", "")),
|
||||
name=str(entry.get("name", "")),
|
||||
params=resolved_params if isinstance(resolved_params, dict) else {},
|
||||
target_folder=str(entry.get("target_folder", "")),
|
||||
sync_frequency=str(entry.get("sync_frequency", "manual")),
|
||||
enabled=bool(entry.get("enabled", True)),
|
||||
)
|
||||
)
|
||||
|
||||
# --- Phase 4: PipelineSettings parsen ---
|
||||
settings = _parse_pipeline_settings(settings_raw)
|
||||
|
||||
return sources, settings
|
||||
|
||||
|
||||
def _parse_pipeline_settings(raw: Any) -> PipelineSettings:
|
||||
"""Parst den settings-Block aus sources.yaml.
|
||||
|
||||
Args:
|
||||
raw: Der rohe dict aus dem YAML.
|
||||
|
||||
Returns:
|
||||
Ein PipelineSettings-Objekt mit Defaults für fehlende Felder.
|
||||
"""
|
||||
if not isinstance(raw, dict):
|
||||
return PipelineSettings()
|
||||
|
||||
# Enrichment settings
|
||||
enrichment_raw = raw.get("enrichment", {})
|
||||
enrichment = EnrichmentSettings()
|
||||
if isinstance(enrichment_raw, dict):
|
||||
# Env vars im enrichment-Block auflösen
|
||||
resolved_enrichment = _resolve_env_vars_recursive(enrichment_raw)
|
||||
if isinstance(resolved_enrichment, dict):
|
||||
enrichment = EnrichmentSettings(
|
||||
provider=str(resolved_enrichment.get("provider", "kiro")),
|
||||
model=str(resolved_enrichment.get("model", "")),
|
||||
confidence_threshold=float(
|
||||
resolved_enrichment.get("confidence_threshold", 0.7)
|
||||
),
|
||||
)
|
||||
|
||||
# OCR settings
|
||||
ocr_raw = raw.get("ocr", {})
|
||||
ocr = OCRSettings()
|
||||
if isinstance(ocr_raw, dict):
|
||||
resolved_ocr = _resolve_env_vars_recursive(ocr_raw)
|
||||
if isinstance(resolved_ocr, dict):
|
||||
ocr = OCRSettings(
|
||||
provider=str(resolved_ocr.get("provider", "kiro")),
|
||||
languages=str(resolved_ocr.get("languages", "deu+eng")),
|
||||
)
|
||||
|
||||
# Git settings
|
||||
git_raw = raw.get("git", {})
|
||||
git = GitSettings()
|
||||
if isinstance(git_raw, dict):
|
||||
git = GitSettings(
|
||||
auto_commit=bool(git_raw.get("auto_commit", True)),
|
||||
)
|
||||
|
||||
return PipelineSettings(
|
||||
create_tasks=bool(raw.get("create_tasks", True)),
|
||||
enrichment=enrichment,
|
||||
ocr=ocr,
|
||||
git=git,
|
||||
)
|
||||
@@ -0,0 +1,692 @@
|
||||
"""Enrichment – KI-gestützte Anreicherung von Wissensartefakten.
|
||||
|
||||
Extrahiert Tags, Kategorien, Entitäten und Action-Items aus
|
||||
Rohartefakten mittels LLM-Aufrufen (Kiro-first) oder regelbasierten
|
||||
Heuristiken als Fallback.
|
||||
|
||||
Requirements: 9.1, 9.2, 9.3, 9.4, 9.5, 9.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionItem:
|
||||
"""An extracted action item from content.
|
||||
|
||||
Attributes:
|
||||
description: The action item text.
|
||||
assignee: Optional person assigned to the action.
|
||||
deadline: Optional ISO date string for the deadline.
|
||||
hash: SHA-256 hash of description for deduplication.
|
||||
"""
|
||||
|
||||
description: str
|
||||
assignee: str | None = None
|
||||
deadline: str | None = None # ISO date string
|
||||
hash: str = "" # for dedup: sha256 of description
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.hash and self.description:
|
||||
self.hash = _compute_action_item_hash(self.description)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichmentResult:
|
||||
"""Output from enrichment analysis.
|
||||
|
||||
Attributes:
|
||||
title: Extracted or generated title.
|
||||
category: Content category (meeting, decision, project, reference, link, inbox).
|
||||
tags: Generated tags describing the content.
|
||||
people: Detected person names (confidence >= 0.7).
|
||||
projects: Detected project references (confidence >= 0.7).
|
||||
action_items: Extracted action items.
|
||||
confidence: Overall confidence score (0.0 - 1.0).
|
||||
enrichment_pending: True if LLM provider was unavailable.
|
||||
"""
|
||||
|
||||
title: str
|
||||
category: str # meeting, decision, project, reference, link, inbox
|
||||
tags: list[str] = field(default_factory=list)
|
||||
people: list[str] = field(default_factory=list)
|
||||
projects: list[str] = field(default_factory=list)
|
||||
action_items: list[ActionItem] = field(default_factory=list)
|
||||
confidence: float = 0.0
|
||||
enrichment_pending: bool = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Minimum confidence threshold for including entities in output.
|
||||
CONFIDENCE_THRESHOLD = 0.7
|
||||
|
||||
#: Valid categories for artifact classification.
|
||||
VALID_CATEGORIES = ("meeting", "decision", "project", "reference", "link", "inbox")
|
||||
|
||||
#: Category detection keywords (German + English).
|
||||
CATEGORY_KEYWORDS: dict[str, list[str]] = {
|
||||
"meeting": [
|
||||
"meeting", "besprechung", "jour fixe", "standup", "stand-up",
|
||||
"sync", "retro", "retrospektive", "daily", "weekly",
|
||||
"protokoll", "minutes", "teilnehmer", "participants",
|
||||
"agenda", "termin",
|
||||
],
|
||||
"decision": [
|
||||
"decision", "entscheidung", "beschluss", "agreed", "decided",
|
||||
"resolution", "entschieden", "festgelegt", "approval",
|
||||
"genehmigung", "freigabe",
|
||||
],
|
||||
"project": [
|
||||
"projekt", "project", "milestone", "meilenstein", "sprint",
|
||||
"release", "roadmap", "backlog", "epic", "feature",
|
||||
"workstream", "initiative",
|
||||
],
|
||||
"reference": [
|
||||
"referenz", "reference", "dokumentation", "documentation",
|
||||
"howto", "how-to", "anleitung", "tutorial", "guide",
|
||||
"handbuch", "manual", "wiki", "spec", "specification",
|
||||
],
|
||||
"link": [
|
||||
"http://", "https://", "www.",
|
||||
],
|
||||
}
|
||||
|
||||
#: Patterns for detecting action items in text.
|
||||
ACTION_ITEM_PATTERNS: list[re.Pattern[str]] = [
|
||||
re.compile(r"^[-*]\s*\[\s*\]\s*(.+)", re.MULTILINE), # - [ ] task
|
||||
re.compile(r"^TODO:\s*(.+)", re.MULTILINE | re.IGNORECASE), # TODO: task
|
||||
re.compile(r"^ACTION:\s*(.+)", re.MULTILINE | re.IGNORECASE), # ACTION: task
|
||||
re.compile(r"^→\s*(.+)", re.MULTILINE), # → task
|
||||
re.compile(r"^=>\s*(.+)", re.MULTILINE), # => task
|
||||
]
|
||||
|
||||
#: Pattern for detecting person names (Vorname Nachname).
|
||||
# Supports Latin letters with diacritics (é, è, ñ, etc.).
|
||||
PERSON_NAME_PATTERN = re.compile(
|
||||
r"(?<![#/\[])(?:^|\s)(@?[A-ZÀ-ÖØ-Þ][a-zà-öø-ÿß]{2,}\s[A-ZÀ-ÖØ-Þ][a-zà-öø-ÿß]{2,})(?:\s|[,;.\-!?]|$)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
#: Pattern for @mentions.
|
||||
AT_MENTION_PATTERN = re.compile(r"@([A-Za-zÄÖÜäöüß]+(?:\s[A-Za-zÄÖÜäöüß]+)?)")
|
||||
|
||||
#: Pattern for detecting project references (e.g., "Projekt XYZ", "Project: ABC").
|
||||
PROJECT_REF_PATTERNS: list[re.Pattern[str]] = [
|
||||
re.compile(r"(?:Projekt|Project)\s*[:\-]?\s*([A-Za-zÄÖÜäöüß0-9][\w\s\-]{2,30})", re.IGNORECASE),
|
||||
re.compile(r"\[\[projects/([^\]]+)\]\]"), # existing wiki links
|
||||
]
|
||||
|
||||
#: ISO date pattern for deadline detection.
|
||||
ISO_DATE_PATTERN = re.compile(
|
||||
r"(?:bis|by|until|deadline:?|fällig:?)\s*(\d{4}-\d{2}-\d{2})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
#: Common stop words to exclude from people detection.
|
||||
_STOP_NAMES: set[str] = {
|
||||
"Action Items", "Quick Capture", "Knowledge Management",
|
||||
"Meeting Notes", "Daily Standup", "Deutsche Bahn",
|
||||
"Sprint Review", "Sprint Planning", "Open Source",
|
||||
"Pull Request", "Merge Request", "Code Review",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper Functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slugify(name: str) -> str:
|
||||
"""Convert a name to a URL/wiki-link slug.
|
||||
|
||||
Converts "André Knie" → "andre-knie", "Max Müller" → "max-mueller".
|
||||
|
||||
Args:
|
||||
name: The name to slugify.
|
||||
|
||||
Returns:
|
||||
Lowercase hyphenated slug suitable for wiki-links.
|
||||
"""
|
||||
# Normalize unicode (decompose then remove combining marks)
|
||||
text = name.strip().lower()
|
||||
|
||||
# German umlaut replacements before NFD decomposition
|
||||
replacements = {
|
||||
"ä": "ae", "ö": "oe", "ü": "ue",
|
||||
"ß": "ss",
|
||||
}
|
||||
for char, repl in replacements.items():
|
||||
text = text.replace(char, repl)
|
||||
|
||||
# Remove remaining diacritics
|
||||
text = unicodedata.normalize("NFD", text)
|
||||
text = "".join(c for c in text if unicodedata.category(c) != "Mn")
|
||||
|
||||
# Replace non-alphanumeric with hyphens
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text)
|
||||
|
||||
# Collapse multiple hyphens and strip leading/trailing
|
||||
text = re.sub(r"-+", "-", text).strip("-")
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _compute_action_item_hash(description: str) -> str:
|
||||
"""Compute SHA-256 hash of an action item description for deduplication.
|
||||
|
||||
Args:
|
||||
description: The action item description text.
|
||||
|
||||
Returns:
|
||||
First 16 characters of hex SHA-256 digest.
|
||||
"""
|
||||
normalized = description.strip().lower()
|
||||
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _detect_category(content: str, source_type: str = "") -> tuple[str, float]:
|
||||
"""Detect the content category using keyword matching.
|
||||
|
||||
Args:
|
||||
content: The text content to classify.
|
||||
source_type: Optional source type hint (e.g., "email", "confluence").
|
||||
|
||||
Returns:
|
||||
Tuple of (category, confidence).
|
||||
"""
|
||||
content_lower = content.lower()
|
||||
|
||||
# Source type can give strong hints
|
||||
source_type_hints: dict[str, str] = {
|
||||
"email": "reference",
|
||||
"chat": "reference",
|
||||
"confluence": "reference",
|
||||
"jira": "project",
|
||||
}
|
||||
|
||||
scores: dict[str, float] = {cat: 0.0 for cat in VALID_CATEGORIES}
|
||||
|
||||
# Count keyword matches per category
|
||||
for category, keywords in CATEGORY_KEYWORDS.items():
|
||||
for keyword in keywords:
|
||||
if keyword in content_lower:
|
||||
scores[category] += 1.0
|
||||
|
||||
# Apply source type hint bonus
|
||||
if source_type and source_type.lower() in source_type_hints:
|
||||
hinted = source_type_hints[source_type.lower()]
|
||||
scores[hinted] += 0.5
|
||||
|
||||
# Check if content is primarily a URL/link
|
||||
stripped = content.strip()
|
||||
if stripped.startswith(("http://", "https://")) and "\n" not in stripped:
|
||||
return "link", 0.9
|
||||
|
||||
# Find best category
|
||||
best_category = max(scores, key=lambda k: scores[k])
|
||||
best_score = scores[best_category]
|
||||
|
||||
if best_score == 0.0:
|
||||
return "inbox", 0.3
|
||||
|
||||
# Normalize confidence (more matches = higher confidence, capped at 0.95)
|
||||
confidence = min(0.95, 0.5 + best_score * 0.1)
|
||||
|
||||
return best_category, confidence
|
||||
|
||||
|
||||
def _extract_people(content: str) -> list[tuple[str, float]]:
|
||||
"""Extract person names from content with confidence scores.
|
||||
|
||||
Detects patterns like "Max Müller", "@andre.knie", "@André Knie".
|
||||
|
||||
Args:
|
||||
content: The text to analyze.
|
||||
|
||||
Returns:
|
||||
List of (name, confidence) tuples.
|
||||
"""
|
||||
people: dict[str, float] = {}
|
||||
|
||||
# Detect @mentions (high confidence)
|
||||
for match in AT_MENTION_PATTERN.finditer(content):
|
||||
name = match.group(1).strip()
|
||||
if len(name) >= 3 and name not in _STOP_NAMES:
|
||||
people[name] = max(people.get(name, 0), 0.9)
|
||||
|
||||
# Detect "Vorname Nachname" patterns
|
||||
for match in PERSON_NAME_PATTERN.finditer(content):
|
||||
name = match.group(1).strip().lstrip("@")
|
||||
if name in _STOP_NAMES:
|
||||
continue
|
||||
if len(name) >= 5: # at least "Xx Yy" length
|
||||
# Lower confidence for bare name patterns (could be titles, places)
|
||||
people[name] = max(people.get(name, 0), 0.75)
|
||||
|
||||
return [(name, conf) for name, conf in people.items()]
|
||||
|
||||
|
||||
def _extract_projects(content: str) -> list[tuple[str, float]]:
|
||||
"""Extract project references from content with confidence scores.
|
||||
|
||||
Args:
|
||||
content: The text to analyze.
|
||||
|
||||
Returns:
|
||||
List of (project_name, confidence) tuples.
|
||||
"""
|
||||
projects: dict[str, float] = {}
|
||||
|
||||
for pattern in PROJECT_REF_PATTERNS:
|
||||
for match in pattern.finditer(content):
|
||||
proj = match.group(1).strip().rstrip(".,;:!?")
|
||||
if len(proj) >= 2:
|
||||
projects[proj] = max(projects.get(proj, 0), 0.8)
|
||||
|
||||
return [(name, conf) for name, conf in projects.items()]
|
||||
|
||||
|
||||
def _extract_action_items(content: str) -> list[ActionItem]:
|
||||
"""Extract action items from content.
|
||||
|
||||
Detects patterns like:
|
||||
- [ ] Do something
|
||||
TODO: Something else
|
||||
ACTION: Assign this task
|
||||
→ Next step
|
||||
|
||||
Args:
|
||||
content: The text to analyze.
|
||||
|
||||
Returns:
|
||||
List of ActionItem instances.
|
||||
"""
|
||||
seen_hashes: set[str] = set()
|
||||
items: list[ActionItem] = []
|
||||
|
||||
for pattern in ACTION_ITEM_PATTERNS:
|
||||
for match in pattern.finditer(content):
|
||||
description = match.group(1).strip()
|
||||
if not description or len(description) < 3:
|
||||
continue
|
||||
|
||||
item_hash = _compute_action_item_hash(description)
|
||||
if item_hash in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(item_hash)
|
||||
|
||||
# Try to extract assignee from description
|
||||
assignee = _extract_assignee(description)
|
||||
|
||||
# Try to extract deadline
|
||||
deadline = _extract_deadline(description, content)
|
||||
|
||||
items.append(ActionItem(
|
||||
description=description,
|
||||
assignee=assignee,
|
||||
deadline=deadline,
|
||||
hash=item_hash,
|
||||
))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _extract_assignee(description: str) -> str | None:
|
||||
"""Try to extract an assignee from action item text.
|
||||
|
||||
Looks for patterns like "@Name" or "(Name)" at the end.
|
||||
|
||||
Args:
|
||||
description: The action item description.
|
||||
|
||||
Returns:
|
||||
Assignee name or None.
|
||||
"""
|
||||
# Check for @mention in description (supports diacritics)
|
||||
at_match = re.search(r"@([\w\u00C0-\u024F]+(?:\s[\w\u00C0-\u024F]+)?)", description)
|
||||
if at_match:
|
||||
return at_match.group(1)
|
||||
|
||||
# Check for (Name) at end
|
||||
paren_match = re.search(
|
||||
r"\(([A-ZÀ-ÖØ-Þ][\w\u00C0-\u024F]+(?:\s[A-ZÀ-ÖØ-Þ][\w\u00C0-\u024F]+)?)\)\s*$",
|
||||
description,
|
||||
)
|
||||
if paren_match:
|
||||
return paren_match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_deadline(description: str, full_content: str = "") -> str | None:
|
||||
"""Try to extract a deadline from action item or surrounding context.
|
||||
|
||||
Args:
|
||||
description: The action item description.
|
||||
full_content: The full document content for context.
|
||||
|
||||
Returns:
|
||||
ISO date string or None.
|
||||
"""
|
||||
# Look in description first
|
||||
date_match = ISO_DATE_PATTERN.search(description)
|
||||
if date_match:
|
||||
return date_match.group(1)
|
||||
|
||||
# Simple date pattern in description
|
||||
simple_date = re.search(r"(\d{4}-\d{2}-\d{2})", description)
|
||||
if simple_date:
|
||||
return simple_date.group(1)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_title(content: str) -> str:
|
||||
"""Extract title from first heading or first line of content.
|
||||
|
||||
Args:
|
||||
content: The text content.
|
||||
|
||||
Returns:
|
||||
Extracted title string.
|
||||
"""
|
||||
lines = content.strip().split("\n")
|
||||
|
||||
for line in lines[:5]: # Check first 5 lines
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Markdown heading
|
||||
heading_match = re.match(r"^#{1,3}\s+(.+)", line)
|
||||
if heading_match:
|
||||
return heading_match.group(1).strip()
|
||||
|
||||
# First non-empty line as fallback title
|
||||
if len(line) > 2:
|
||||
# Truncate very long first lines
|
||||
return line[:80].rstrip(".,;:!?")
|
||||
|
||||
return "Untitled"
|
||||
|
||||
|
||||
def _extract_tags(content: str, category: str) -> list[str]:
|
||||
"""Extract relevant tags from content using keyword frequency.
|
||||
|
||||
Args:
|
||||
content: The text content to analyze.
|
||||
category: The detected category (used as base tag).
|
||||
|
||||
Returns:
|
||||
List of tag strings.
|
||||
"""
|
||||
tags: list[str] = []
|
||||
|
||||
# Add category as a tag if meaningful
|
||||
if category and category != "inbox":
|
||||
tags.append(category)
|
||||
|
||||
content_lower = content.lower()
|
||||
|
||||
# Domain-specific tag detection
|
||||
tech_keywords = {
|
||||
"python", "java", "typescript", "docker", "kubernetes", "k8s",
|
||||
"api", "rest", "graphql", "ci/cd", "pipeline", "terraform",
|
||||
"aws", "azure", "cloud", "microservice", "frontend", "backend",
|
||||
"database", "sql", "mongodb", "redis",
|
||||
}
|
||||
|
||||
bahn_keywords = {
|
||||
"infrago", "db", "confluence", "jira", "openshift",
|
||||
"pipeship", "cnp", "einfachbahn",
|
||||
}
|
||||
|
||||
for kw in tech_keywords:
|
||||
if kw in content_lower:
|
||||
tags.append(kw)
|
||||
|
||||
for kw in bahn_keywords:
|
||||
if kw in content_lower:
|
||||
tags.append(kw)
|
||||
|
||||
# Limit tags to reasonable number
|
||||
return tags[:10]
|
||||
|
||||
|
||||
def generate_wiki_links(people: list[str], projects: list[str]) -> dict[str, list[str]]:
|
||||
"""Generate wiki-links for detected entities.
|
||||
|
||||
Args:
|
||||
people: List of detected person names.
|
||||
projects: List of detected project names.
|
||||
|
||||
Returns:
|
||||
Dict with 'people' and 'projects' wiki-link lists.
|
||||
"""
|
||||
people_links = [f"[[people/{_slugify(name)}]]" for name in people]
|
||||
project_links = [f"[[projects/{_slugify(proj)}]]" for proj in projects]
|
||||
return {"people": people_links, "projects": project_links}
|
||||
|
||||
|
||||
def generate_action_items_section(action_items: list[ActionItem]) -> str:
|
||||
"""Generate a markdown section for action items.
|
||||
|
||||
Args:
|
||||
action_items: List of ActionItem instances.
|
||||
|
||||
Returns:
|
||||
Markdown string with ## Action Items heading and list.
|
||||
"""
|
||||
if not action_items:
|
||||
return ""
|
||||
|
||||
lines = ["## Action Items", ""]
|
||||
for item in action_items:
|
||||
parts = [f"- [ ] {item.description}"]
|
||||
annotations: list[str] = []
|
||||
if item.assignee:
|
||||
annotations.append(f"@{item.assignee}")
|
||||
if item.deadline:
|
||||
annotations.append(f"bis {item.deadline}")
|
||||
if annotations:
|
||||
parts[0] += f" ({', '.join(annotations)})"
|
||||
lines.append(parts[0])
|
||||
|
||||
lines.append("") # trailing newline
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EnrichmentAgent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EnrichmentAgent:
|
||||
"""Content analysis and enrichment agent.
|
||||
|
||||
Provider-Strategie (Priorität):
|
||||
1. Kiro AI-Agent (primär) – nutzt den laufenden Orchestrator-Agenten direkt,
|
||||
keine externe Abhängigkeit, kein API-Key nötig
|
||||
2. Regelbasierte Heuristiken als Fallback wenn kein LLM verfügbar
|
||||
|
||||
Im aktuellen MVP werden nur die regelbasierten Heuristiken verwendet.
|
||||
Die LLM-Integration (Kiro/LiteLLM) wird als Stub vorgehalten.
|
||||
"""
|
||||
|
||||
def __init__(self, provider: str = "kiro", model: str = "") -> None:
|
||||
"""Initialize with LLM config.
|
||||
|
||||
Args:
|
||||
provider: LLM provider ('kiro' or 'litellm'). Defaults to 'kiro'.
|
||||
model: Model identifier (empty = provider default).
|
||||
"""
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
self._provider_available = self._check_provider_available()
|
||||
|
||||
def _check_provider_available(self) -> bool:
|
||||
"""Check if the configured LLM provider is available.
|
||||
|
||||
For now, rule-based heuristics are always available.
|
||||
The Kiro/LiteLLM check is a stub for future implementation.
|
||||
|
||||
Returns:
|
||||
True if provider is available, False otherwise.
|
||||
"""
|
||||
# Rule-based heuristics are always available
|
||||
# When LLM providers are implemented, this will actually check connectivity
|
||||
if self.provider == "kiro":
|
||||
# Kiro provider: in MVP we fall back to heuristics which are always available
|
||||
return True
|
||||
elif self.provider == "litellm":
|
||||
# LiteLLM would require env vars LLM_PROVIDER, LLM_MODEL
|
||||
# Stub: not available in MVP
|
||||
return False
|
||||
return False
|
||||
|
||||
def enrich(self, content: str, source_type: str = "") -> EnrichmentResult:
|
||||
"""Analyze content and generate metadata.
|
||||
|
||||
Uses rule-based heuristics to extract:
|
||||
- Title from first heading or first line
|
||||
- Category from keyword matching
|
||||
- Tags from domain keywords
|
||||
- People from name patterns and @mentions
|
||||
- Projects from explicit references
|
||||
- Action items from TODO/checkbox patterns
|
||||
|
||||
Only entities with confidence >= 0.7 are included in the output.
|
||||
|
||||
Args:
|
||||
content: The text content to analyze.
|
||||
source_type: Optional hint about the source type.
|
||||
|
||||
Returns:
|
||||
EnrichmentResult with extracted metadata.
|
||||
"""
|
||||
if not content or not content.strip():
|
||||
return EnrichmentResult(
|
||||
title="Untitled",
|
||||
category="inbox",
|
||||
confidence=0.0,
|
||||
enrichment_pending=not self._provider_available,
|
||||
)
|
||||
|
||||
# If provider is not available, return minimal result with pending flag
|
||||
if not self._provider_available:
|
||||
title = _extract_title(content)
|
||||
return EnrichmentResult(
|
||||
title=title,
|
||||
category="inbox",
|
||||
confidence=0.0,
|
||||
enrichment_pending=True,
|
||||
)
|
||||
|
||||
# Extract title
|
||||
title = _extract_title(content)
|
||||
|
||||
# Detect category
|
||||
category, cat_confidence = _detect_category(content, source_type)
|
||||
|
||||
# Extract tags
|
||||
tags = _extract_tags(content, category)
|
||||
|
||||
# Extract people (apply confidence threshold)
|
||||
raw_people = _extract_people(content)
|
||||
people = [name for name, conf in raw_people if conf >= CONFIDENCE_THRESHOLD]
|
||||
|
||||
# Extract projects (apply confidence threshold)
|
||||
raw_projects = _extract_projects(content)
|
||||
projects = [name for name, conf in raw_projects if conf >= CONFIDENCE_THRESHOLD]
|
||||
|
||||
# Extract action items
|
||||
action_items = _extract_action_items(content)
|
||||
|
||||
# Compute overall confidence
|
||||
# Based on: how much structure was found in the content
|
||||
signals = [
|
||||
cat_confidence,
|
||||
min(1.0, len(tags) * 0.2) if tags else 0.0,
|
||||
0.8 if people else 0.0,
|
||||
0.8 if action_items else 0.0,
|
||||
]
|
||||
overall_confidence = sum(signals) / len(signals) if signals else 0.3
|
||||
|
||||
return EnrichmentResult(
|
||||
title=title,
|
||||
category=category,
|
||||
tags=tags,
|
||||
people=people,
|
||||
projects=projects,
|
||||
action_items=action_items,
|
||||
confidence=round(overall_confidence, 2),
|
||||
enrichment_pending=False,
|
||||
)
|
||||
|
||||
def classify_relevance(self, content: str) -> float:
|
||||
"""Score content relevance (0.0 = irrelevant, 1.0 = highly relevant).
|
||||
|
||||
Simple heuristic based on:
|
||||
- Content length (very short = likely low relevance)
|
||||
- Structural elements (headings, lists, links)
|
||||
- Domain keywords presence
|
||||
- Action items presence
|
||||
|
||||
Args:
|
||||
content: The text content to score.
|
||||
|
||||
Returns:
|
||||
Float between 0.0 and 1.0 indicating relevance.
|
||||
"""
|
||||
if not content or not content.strip():
|
||||
return 0.0
|
||||
|
||||
score = 0.0
|
||||
content_stripped = content.strip()
|
||||
|
||||
# Length-based scoring
|
||||
length = len(content_stripped)
|
||||
if length < 20:
|
||||
score += 0.1
|
||||
elif length < 100:
|
||||
score += 0.3
|
||||
elif length < 500:
|
||||
score += 0.5
|
||||
elif length < 2000:
|
||||
score += 0.7
|
||||
else:
|
||||
score += 0.8
|
||||
|
||||
# Structural elements bonus
|
||||
has_headings = bool(re.search(r"^#+\s", content, re.MULTILINE))
|
||||
has_lists = bool(re.search(r"^[-*]\s", content, re.MULTILINE))
|
||||
has_links = bool(re.search(r"\[.+?\]\(.+?\)", content))
|
||||
|
||||
if has_headings:
|
||||
score += 0.1
|
||||
if has_lists:
|
||||
score += 0.05
|
||||
if has_links:
|
||||
score += 0.05
|
||||
|
||||
# Action items bonus
|
||||
for pattern in ACTION_ITEM_PATTERNS:
|
||||
if pattern.search(content):
|
||||
score += 0.1
|
||||
break
|
||||
|
||||
# Cap at 1.0
|
||||
return min(1.0, round(score, 2))
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Git-Integration – Automatisches Committen von Wissensänderungen.
|
||||
|
||||
Erstellt strukturierte Git-Commits nach erfolgreicher Ingestion
|
||||
mit konventionellen Commit-Messages.
|
||||
|
||||
Commit-Message-Format:
|
||||
knowledge({context}): {action} {count} artifacts from {source}
|
||||
knowledge({context}): capture "{title}"
|
||||
|
||||
Verwendet subprocess für Git-Aufrufe (kein gitpython).
|
||||
Fehler werden graceful behandelt: Warnung loggen, Dateien beibehalten.
|
||||
|
||||
Requirements: 16.1, 16.2, 16.3, 16.4, 16.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_commit_message(
|
||||
context: str,
|
||||
action: str,
|
||||
count: int,
|
||||
source: str,
|
||||
) -> str:
|
||||
"""Generiert eine konventionelle Commit-Message für Knowledge-Änderungen.
|
||||
|
||||
Format: knowledge({context}): {action} {count} artifacts from {source}
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (bahn, dhive, privat).
|
||||
action: Die durchgeführte Aktion (ingest, capture, update, etc.).
|
||||
count: Anzahl der verarbeiteten Artefakte.
|
||||
source: Die Quelle der Artefakte (confluence, jira, inbox, etc.).
|
||||
|
||||
Returns:
|
||||
Die formatierte Commit-Message.
|
||||
"""
|
||||
return f"knowledge({context}): {action} {count} artifacts from {source}"
|
||||
|
||||
|
||||
def generate_capture_commit_message(context: str, title: str) -> str:
|
||||
"""Generiert eine Commit-Message für Quick-Capture-Artefakte.
|
||||
|
||||
Format: knowledge({context}): capture "{title}"
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (bahn, dhive, privat).
|
||||
title: Der Titel des erfassten Artefakts.
|
||||
|
||||
Returns:
|
||||
Die formatierte Commit-Message.
|
||||
"""
|
||||
return f'knowledge({context}): capture "{title}"'
|
||||
|
||||
|
||||
def auto_commit(
|
||||
context: str,
|
||||
action: str,
|
||||
count: int,
|
||||
source: str,
|
||||
paths: list[Path],
|
||||
*,
|
||||
no_commit: bool = False,
|
||||
auto_commit_enabled: bool = True,
|
||||
cwd: Path | None = None,
|
||||
) -> str:
|
||||
"""Staged und committed Änderungen an den angegebenen Pfaden.
|
||||
|
||||
Respektiert --no-commit-Flag und git.auto_commit-Config.
|
||||
Behandelt Git-Fehler graceful (Warnung, Dateien bleiben erhalten).
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (bahn, dhive, privat).
|
||||
action: Die durchgeführte Aktion (ingest, capture, etc.).
|
||||
count: Anzahl der verarbeiteten Artefakte.
|
||||
source: Die Quelle der Artefakte.
|
||||
paths: Liste der zu committenden Dateipfade.
|
||||
no_commit: Wenn True, wird kein Commit erzeugt (--no-commit Flag).
|
||||
auto_commit_enabled: Wert aus git.auto_commit in sources.yaml.
|
||||
cwd: Arbeitsverzeichnis für Git-Befehle. Wenn None, wird das
|
||||
übergeordnete Verzeichnis des ersten Pfades verwendet.
|
||||
|
||||
Returns:
|
||||
Den Commit-SHA bei Erfolg, oder einen leeren String wenn kein
|
||||
Commit erzeugt wurde (no_commit, disabled, Fehler, keine Pfade).
|
||||
"""
|
||||
# Keine Pfade → nichts zu committen
|
||||
if not paths:
|
||||
logger.debug("auto_commit: Keine Pfade angegeben, überspringe Commit.")
|
||||
return ""
|
||||
|
||||
# --no-commit Flag hat höchste Priorität
|
||||
if no_commit:
|
||||
logger.info("auto_commit: --no-commit gesetzt, überspringe Git-Commit.")
|
||||
return ""
|
||||
|
||||
# git.auto_commit aus Config prüfen
|
||||
if not auto_commit_enabled:
|
||||
logger.info(
|
||||
"auto_commit: git.auto_commit ist deaktiviert, überspringe Git-Commit."
|
||||
)
|
||||
return ""
|
||||
|
||||
# Arbeitsverzeichnis bestimmen
|
||||
if cwd is None:
|
||||
cwd = paths[0].parent
|
||||
|
||||
# Commit-Message generieren
|
||||
message = generate_commit_message(context, action, count, source)
|
||||
|
||||
return _execute_git_commit(paths, message, cwd)
|
||||
|
||||
|
||||
def auto_commit_capture(
|
||||
context: str,
|
||||
title: str,
|
||||
paths: list[Path],
|
||||
*,
|
||||
no_commit: bool = False,
|
||||
auto_commit_enabled: bool = True,
|
||||
cwd: Path | None = None,
|
||||
) -> str:
|
||||
"""Staged und committed ein Quick-Capture-Artefakt.
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (bahn, dhive, privat).
|
||||
title: Der Titel des erfassten Artefakts.
|
||||
paths: Liste der zu committenden Dateipfade.
|
||||
no_commit: Wenn True, wird kein Commit erzeugt.
|
||||
auto_commit_enabled: Wert aus git.auto_commit in sources.yaml.
|
||||
cwd: Arbeitsverzeichnis für Git-Befehle.
|
||||
|
||||
Returns:
|
||||
Den Commit-SHA bei Erfolg, oder einen leeren String.
|
||||
"""
|
||||
if not paths:
|
||||
return ""
|
||||
|
||||
if no_commit:
|
||||
logger.info("auto_commit_capture: --no-commit gesetzt, überspringe.")
|
||||
return ""
|
||||
|
||||
if not auto_commit_enabled:
|
||||
logger.info("auto_commit_capture: git.auto_commit deaktiviert, überspringe.")
|
||||
return ""
|
||||
|
||||
if cwd is None:
|
||||
cwd = paths[0].parent
|
||||
|
||||
message = generate_capture_commit_message(context, title)
|
||||
|
||||
return _execute_git_commit(paths, message, cwd)
|
||||
|
||||
|
||||
def _execute_git_commit(
|
||||
paths: list[Path],
|
||||
message: str,
|
||||
cwd: Path,
|
||||
) -> str:
|
||||
"""Führt git add + git commit aus.
|
||||
|
||||
Bei Fehlern wird eine Warnung geloggt und ein leerer String zurückgegeben.
|
||||
Die erstellten Dateien bleiben in jedem Fall erhalten.
|
||||
|
||||
Args:
|
||||
paths: Zu stagenden Dateipfade.
|
||||
message: Die Commit-Message.
|
||||
cwd: Arbeitsverzeichnis für Git-Befehle.
|
||||
|
||||
Returns:
|
||||
Den Commit-SHA bei Erfolg, oder einen leeren String bei Fehler.
|
||||
"""
|
||||
try:
|
||||
# Stage files
|
||||
str_paths = [str(p) for p in paths]
|
||||
_run_git(["add", "--"] + str_paths, cwd=cwd)
|
||||
|
||||
# Commit
|
||||
_run_git(["commit", "-m", message], cwd=cwd)
|
||||
|
||||
# SHA des letzten Commits holen
|
||||
result = _run_git(["rev-parse", "HEAD"], cwd=cwd)
|
||||
sha = result.stdout.strip()
|
||||
logger.info(f"Git-Commit erstellt: {sha[:8]} – {message}")
|
||||
return sha
|
||||
|
||||
except GitCommandError as e:
|
||||
logger.warning(
|
||||
f"Git-Commit fehlgeschlagen: {e}. "
|
||||
"Dateien wurden erstellt, aber nicht committed."
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def _run_git(
|
||||
args: list[str],
|
||||
cwd: Path,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
"""Führt einen Git-Befehl als Subprocess aus.
|
||||
|
||||
Args:
|
||||
args: Git-Subcommand und Argumente (ohne 'git' Präfix).
|
||||
cwd: Arbeitsverzeichnis.
|
||||
|
||||
Returns:
|
||||
Das CompletedProcess-Objekt bei Erfolg.
|
||||
|
||||
Raises:
|
||||
GitCommandError: Wenn der Git-Befehl fehlschlägt.
|
||||
"""
|
||||
cmd = ["git"] + args
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(cwd),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise GitCommandError(
|
||||
"Git ist nicht installiert oder nicht im PATH verfügbar."
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise GitCommandError(f"Git-Befehl hat Timeout überschritten: {cmd}")
|
||||
except OSError as e:
|
||||
raise GitCommandError(f"OS-Fehler beim Ausführen von Git: {e}")
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = result.stderr.strip()
|
||||
raise GitCommandError(
|
||||
f"Git-Befehl fehlgeschlagen (exit {result.returncode}): "
|
||||
f"{' '.join(cmd)}\n{stderr}"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class GitCommandError(Exception):
|
||||
"""Wird bei fehlgeschlagenen Git-Befehlen ausgelöst.
|
||||
|
||||
Diese Exception wird intern in _execute_git_commit abgefangen
|
||||
und in eine Warnung umgewandelt.
|
||||
"""
|
||||
|
||||
pass
|
||||
@@ -0,0 +1,715 @@
|
||||
"""IngestionPipeline – Orchestrator für die Wissensaufnahme.
|
||||
|
||||
Koordiniert den gesamten Ingestion-Workflow: Quellen laden, extrahieren,
|
||||
anreichern, speichern und indexieren.
|
||||
|
||||
Requirements: 2.2, 2.3, 2.4, 2.5, 2.6, 3.4, 3.5, 3.6, 12.3
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
Artifact,
|
||||
ArtifactMetadata,
|
||||
compute_content_hash,
|
||||
write_artifact,
|
||||
)
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.ingestion.config import (
|
||||
PipelineSettings,
|
||||
SourceConfig,
|
||||
load_sources_config,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.enrichment import (
|
||||
ActionItem,
|
||||
EnrichmentAgent,
|
||||
EnrichmentResult,
|
||||
)
|
||||
from monorepo.knowledge.ingestion.git_integration import auto_commit
|
||||
from monorepo.knowledge.ingestion.router import ContextRouter, category_to_folder
|
||||
from monorepo.knowledge.integrations.orgmylife import OrgMyLifeIntegration
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestError:
|
||||
"""Structured error from ingestion pipeline."""
|
||||
|
||||
source: str
|
||||
error_type: str
|
||||
message: str
|
||||
timestamp: str = ""
|
||||
retry: bool = False
|
||||
artifact_path: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.timestamp:
|
||||
self.timestamp = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestionResult:
|
||||
"""Result of a full ingestion run."""
|
||||
|
||||
context: str
|
||||
processed: int = 0
|
||||
updated: int = 0
|
||||
skipped: int = 0
|
||||
errors: list[IngestError] = field(default_factory=list)
|
||||
tasks_created: int = 0
|
||||
commit_sha: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Maps source type names to SourceStrategy classes.
|
||||
# Populated by register_source_strategy() or by the pipeline at startup.
|
||||
_SOURCE_REGISTRY: dict[str, type[SourceStrategy]] = {}
|
||||
|
||||
|
||||
def register_source_strategy(source_type: str, strategy_class: type[SourceStrategy]) -> None:
|
||||
"""Register a source strategy class for a given source type.
|
||||
|
||||
Args:
|
||||
source_type: The source type name (e.g., 'markdown', 'confluence').
|
||||
strategy_class: The SourceStrategy subclass to use for this type.
|
||||
"""
|
||||
_SOURCE_REGISTRY[source_type] = strategy_class
|
||||
|
||||
|
||||
def get_source_strategy(source_type: str) -> SourceStrategy | None:
|
||||
"""Get an instantiated source strategy for the given type.
|
||||
|
||||
Args:
|
||||
source_type: The source type name.
|
||||
|
||||
Returns:
|
||||
An instance of the registered strategy, or None if not registered.
|
||||
"""
|
||||
strategy_class = _SOURCE_REGISTRY.get(source_type)
|
||||
if strategy_class is None:
|
||||
return None
|
||||
return strategy_class()
|
||||
|
||||
|
||||
def register_default_strategies() -> None:
|
||||
"""Register all built-in source strategies.
|
||||
|
||||
This is called automatically when an IngestionPipeline is created,
|
||||
ensuring all standard source types are available.
|
||||
"""
|
||||
from monorepo.knowledge.sources.chat import ChatSource
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource
|
||||
from monorepo.knowledge.sources.email import EmailSource
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.knowledge.sources.pdf import PDFSource
|
||||
from monorepo.knowledge.sources.wissensdatenbank import WissensdatenbankSource
|
||||
|
||||
defaults: dict[str, type[SourceStrategy]] = {
|
||||
"markdown": MarkdownSource,
|
||||
"pdf": PDFSource,
|
||||
"confluence": ConfluenceSource,
|
||||
"jira": JiraSource,
|
||||
"email": EmailSource,
|
||||
"chat": ChatSource,
|
||||
"wissensdatenbank": WissensdatenbankSource,
|
||||
}
|
||||
|
||||
for source_type, strategy_class in defaults.items():
|
||||
if source_type not in _SOURCE_REGISTRY:
|
||||
_SOURCE_REGISTRY[source_type] = strategy_class
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IngestionPipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class IngestionPipeline:
|
||||
"""Orchestrates the full ingestion workflow.
|
||||
|
||||
Pipeline steps:
|
||||
1. Load sources.yaml configuration
|
||||
2. For each enabled source: extract artifacts via SourceStrategy
|
||||
3. Enrich artifacts (titles, categories, tags, entities)
|
||||
4. Store artifacts in the correct knowledge folder
|
||||
5. Update YAML index
|
||||
6. Optionally commit changes via git
|
||||
|
||||
Fail-forward: individual source failures are logged and skipped,
|
||||
remaining sources continue processing.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
monorepo_root: Path,
|
||||
context: str,
|
||||
dry_run: bool = False,
|
||||
no_commit: bool = False,
|
||||
enrichment_agent: EnrichmentAgent | None = None,
|
||||
source_strategies: dict[str, SourceStrategy] | None = None,
|
||||
) -> None:
|
||||
"""Initialize the ingestion pipeline.
|
||||
|
||||
Args:
|
||||
monorepo_root: Root path of the monorepo.
|
||||
context: Target context (bahn, dhive, privat).
|
||||
dry_run: If True, do not write files.
|
||||
no_commit: If True, skip git commit.
|
||||
enrichment_agent: Optional custom enrichment agent (for testing).
|
||||
source_strategies: Optional custom strategy mapping (for testing).
|
||||
"""
|
||||
self.monorepo_root = monorepo_root
|
||||
self.context = context
|
||||
self.dry_run = dry_run
|
||||
self.no_commit = no_commit
|
||||
self.router = ContextRouter(monorepo_root)
|
||||
self.enrichment_agent = enrichment_agent or EnrichmentAgent()
|
||||
self._source_strategies = source_strategies or {}
|
||||
|
||||
# Ensure all built-in source strategies are registered
|
||||
register_default_strategies()
|
||||
|
||||
@property
|
||||
def knowledge_path(self) -> Path:
|
||||
"""Path to the context's knowledge folder."""
|
||||
return self.monorepo_root / self.context / "knowledge"
|
||||
|
||||
@property
|
||||
def index_path(self) -> Path:
|
||||
"""Path to the context's YAML index."""
|
||||
return self.knowledge_path / "_index.yaml"
|
||||
|
||||
@property
|
||||
def sources_config_path(self) -> Path:
|
||||
"""Path to the context's sources.yaml."""
|
||||
return self.knowledge_path / "sources.yaml"
|
||||
|
||||
def _get_strategy(self, source_type: str) -> SourceStrategy | None:
|
||||
"""Get a source strategy, checking local overrides first.
|
||||
|
||||
Args:
|
||||
source_type: The source type name.
|
||||
|
||||
Returns:
|
||||
A SourceStrategy instance or None.
|
||||
"""
|
||||
if source_type in self._source_strategies:
|
||||
return self._source_strategies[source_type]
|
||||
return get_source_strategy(source_type)
|
||||
|
||||
def _load_index(self) -> YAMLIndex:
|
||||
"""Load the YAML index for this context."""
|
||||
index = YAMLIndex(self.index_path)
|
||||
index.load()
|
||||
return index
|
||||
|
||||
def _is_unchanged(self, artifact: Artifact, index: YAMLIndex) -> bool:
|
||||
"""Check if an artifact is unchanged based on content hash.
|
||||
|
||||
Args:
|
||||
artifact: The artifact to check.
|
||||
index: The current YAML index.
|
||||
|
||||
Returns:
|
||||
True if the artifact's content hash matches the index entry.
|
||||
"""
|
||||
content_hash = compute_content_hash(artifact.content)
|
||||
artifact_id = self._compute_artifact_id(artifact)
|
||||
existing = index.get_entry(artifact_id)
|
||||
if existing and existing.content_hash == content_hash:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _compute_artifact_id(self, artifact: Artifact) -> str:
|
||||
"""Compute a unique ID for an artifact.
|
||||
|
||||
Format: {context}/{type}/{filename_stem}
|
||||
|
||||
Args:
|
||||
artifact: The artifact.
|
||||
|
||||
Returns:
|
||||
String ID for the artifact.
|
||||
"""
|
||||
stem = artifact.file_path.stem if artifact.file_path.stem else "untitled"
|
||||
art_type = artifact.metadata.type or artifact.metadata.category or "inbox"
|
||||
return f"{self.context}/{art_type}/{stem}"
|
||||
|
||||
def _enrich_artifact(self, artifact: Artifact) -> Artifact:
|
||||
"""Enrich an artifact with metadata from the enrichment agent.
|
||||
|
||||
Args:
|
||||
artifact: The raw artifact.
|
||||
|
||||
Returns:
|
||||
The artifact with enriched metadata.
|
||||
"""
|
||||
result = self.enrichment_agent.enrich(
|
||||
artifact.content, artifact.metadata.source.get("type", "")
|
||||
)
|
||||
|
||||
# Apply enrichment results to metadata
|
||||
if not artifact.metadata.title or artifact.metadata.title == "Untitled":
|
||||
artifact.metadata.title = result.title
|
||||
|
||||
if not artifact.metadata.category:
|
||||
artifact.metadata.category = result.category
|
||||
|
||||
if not artifact.metadata.tags:
|
||||
artifact.metadata.tags = result.tags
|
||||
|
||||
if not artifact.metadata.people:
|
||||
artifact.metadata.people = [
|
||||
f"[[people/{_slugify_simple(p)}]]" for p in result.people
|
||||
]
|
||||
|
||||
if not artifact.metadata.projects:
|
||||
artifact.metadata.projects = [
|
||||
f"[[projects/{_slugify_simple(p)}]]" for p in result.projects
|
||||
]
|
||||
|
||||
artifact.metadata.enrichment_pending = result.enrichment_pending
|
||||
|
||||
# Compute content hash
|
||||
artifact.metadata.content_hash = compute_content_hash(artifact.content)
|
||||
|
||||
# Set source context
|
||||
if not artifact.metadata.source_context:
|
||||
artifact.metadata.source_context = self.context
|
||||
|
||||
return artifact
|
||||
|
||||
def _store_artifact(self, artifact: Artifact) -> Path:
|
||||
"""Store an artifact in the knowledge folder.
|
||||
|
||||
Args:
|
||||
artifact: The enriched artifact.
|
||||
|
||||
Returns:
|
||||
The path where the artifact was stored.
|
||||
"""
|
||||
category = artifact.metadata.category or "inbox"
|
||||
folder = category_to_folder(category)
|
||||
target_dir = self.knowledge_path / folder
|
||||
|
||||
if not self.dry_run:
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Determine filename
|
||||
filename = artifact.file_path.name
|
||||
if not filename:
|
||||
filename = "untitled.md"
|
||||
|
||||
target_path = target_dir / filename
|
||||
|
||||
if not self.dry_run:
|
||||
# Write the artifact file
|
||||
from monorepo.knowledge.artifact import _generate_frontmatter
|
||||
|
||||
frontmatter = _generate_frontmatter(artifact.metadata)
|
||||
content = frontmatter + artifact.content
|
||||
target_path.write_text(content, encoding="utf-8")
|
||||
|
||||
artifact.file_path = target_path
|
||||
return target_path
|
||||
|
||||
def _update_index(self, artifact: Artifact, index: YAMLIndex) -> None:
|
||||
"""Update the YAML index with the artifact's metadata.
|
||||
|
||||
Args:
|
||||
artifact: The processed artifact.
|
||||
index: The YAML index to update.
|
||||
"""
|
||||
artifact_id = self._compute_artifact_id(artifact)
|
||||
|
||||
entry = IndexEntry(
|
||||
id=artifact_id,
|
||||
title=artifact.metadata.title,
|
||||
type=artifact.metadata.type or artifact.metadata.category or "inbox",
|
||||
tags=artifact.metadata.tags,
|
||||
scope=self.context,
|
||||
summary="",
|
||||
path=str(artifact.file_path),
|
||||
content_hash=artifact.metadata.content_hash,
|
||||
links=[link.target for link in artifact.metadata.links],
|
||||
)
|
||||
|
||||
index.update_entry(entry)
|
||||
|
||||
def run(self) -> IngestionResult:
|
||||
"""Execute full pipeline: load config → extract → enrich → store → index → commit → tasks.
|
||||
|
||||
Returns:
|
||||
IngestionResult with processing statistics.
|
||||
"""
|
||||
result = IngestionResult(context=self.context)
|
||||
|
||||
# Load index
|
||||
index = self._load_index()
|
||||
|
||||
# Load source configurations
|
||||
sources: list[SourceConfig] = []
|
||||
settings = PipelineSettings()
|
||||
|
||||
if self.sources_config_path.exists():
|
||||
try:
|
||||
sources, settings = load_sources_config(self.sources_config_path)
|
||||
except Exception as e:
|
||||
result.errors.append(IngestError(
|
||||
source="sources.yaml",
|
||||
error_type="config",
|
||||
message=str(e),
|
||||
))
|
||||
return result
|
||||
|
||||
# Track written paths for git commit
|
||||
written_paths: list[Path] = []
|
||||
|
||||
# Track action items for OrgMyLife task creation
|
||||
all_action_items: list[tuple[str, list[ActionItem], str]] = []
|
||||
|
||||
# Process each enabled source
|
||||
for source_config in sources:
|
||||
if not source_config.enabled:
|
||||
continue
|
||||
|
||||
strategy = self._get_strategy(source_config.type)
|
||||
if strategy is None:
|
||||
logger.warning(
|
||||
f"No strategy registered for source type '{source_config.type}'"
|
||||
)
|
||||
result.errors.append(IngestError(
|
||||
source=source_config.name,
|
||||
error_type="config",
|
||||
message=f"No strategy for type '{source_config.type}'",
|
||||
))
|
||||
continue
|
||||
|
||||
# Extract artifacts from source
|
||||
try:
|
||||
extraction = strategy.extract(source_config, self.context)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Source '{source_config.name}' failed: {e}"
|
||||
)
|
||||
result.errors.append(IngestError(
|
||||
source=source_config.name,
|
||||
error_type="extraction",
|
||||
message=str(e),
|
||||
retry=True,
|
||||
))
|
||||
continue
|
||||
|
||||
# Record extraction errors
|
||||
for src_error in extraction.errors:
|
||||
result.errors.append(IngestError(
|
||||
source=src_error.source_name,
|
||||
error_type=src_error.error_type,
|
||||
message=src_error.message,
|
||||
timestamp=src_error.timestamp,
|
||||
retry=src_error.retry,
|
||||
))
|
||||
|
||||
result.skipped += extraction.skipped
|
||||
|
||||
# Process each extracted artifact
|
||||
for artifact in extraction.artifacts:
|
||||
# Check if unchanged (idempotent ingestion)
|
||||
if self._is_unchanged(artifact, index):
|
||||
result.skipped += 1
|
||||
continue
|
||||
|
||||
# Enrich
|
||||
artifact = self._enrich_artifact(artifact)
|
||||
|
||||
# Collect action items for OrgMyLife task creation
|
||||
enrichment_result = self.enrichment_agent.enrich(
|
||||
artifact.content, artifact.metadata.source.get("type", "")
|
||||
)
|
||||
if enrichment_result.action_items:
|
||||
artifact_id = self._compute_artifact_id(artifact)
|
||||
all_action_items.append((
|
||||
artifact_id,
|
||||
enrichment_result.action_items,
|
||||
str(artifact.file_path),
|
||||
))
|
||||
|
||||
# Store
|
||||
if not self.dry_run:
|
||||
stored_path = self._store_artifact(artifact)
|
||||
written_paths.append(stored_path)
|
||||
|
||||
# Update index
|
||||
self._update_index(artifact, index)
|
||||
|
||||
result.processed += 1
|
||||
|
||||
# Save index
|
||||
if not self.dry_run:
|
||||
index.save()
|
||||
written_paths.append(self.index_path)
|
||||
|
||||
# Git auto-commit (Requirement 16.1)
|
||||
if not self.dry_run and written_paths:
|
||||
commit_sha = auto_commit(
|
||||
context=self.context,
|
||||
action="ingest",
|
||||
count=result.processed,
|
||||
source="pipeline",
|
||||
paths=written_paths,
|
||||
no_commit=self.no_commit,
|
||||
auto_commit_enabled=settings.git.auto_commit,
|
||||
cwd=self.monorepo_root,
|
||||
)
|
||||
result.commit_sha = commit_sha
|
||||
|
||||
# OrgMyLife task creation for action items (Requirement 10.1)
|
||||
if not self.dry_run and settings.create_tasks and all_action_items:
|
||||
try:
|
||||
orgmylife = OrgMyLifeIntegration(
|
||||
context_path=self.knowledge_path,
|
||||
enabled=settings.create_tasks,
|
||||
)
|
||||
for artifact_id, action_items, source_path in all_action_items:
|
||||
mappings = orgmylife.create_tasks(
|
||||
artifact_id=artifact_id,
|
||||
action_items=action_items,
|
||||
source_path=source_path,
|
||||
)
|
||||
result.tasks_created += len(mappings)
|
||||
except Exception as e:
|
||||
logger.warning(f"OrgMyLife task creation failed: {e}")
|
||||
result.errors.append(IngestError(
|
||||
source="orgmylife",
|
||||
error_type="integration",
|
||||
message=str(e),
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
def run_with_artifacts(
|
||||
self,
|
||||
artifacts: list[Artifact],
|
||||
source_configs: list[SourceConfig] | None = None,
|
||||
) -> IngestionResult:
|
||||
"""Run pipeline with pre-extracted artifacts (for testing/programmatic use).
|
||||
|
||||
Args:
|
||||
artifacts: Pre-extracted artifacts to process.
|
||||
source_configs: Optional source configs (not used for extraction).
|
||||
|
||||
Returns:
|
||||
IngestionResult with processing statistics.
|
||||
"""
|
||||
result = IngestionResult(context=self.context)
|
||||
index = self._load_index()
|
||||
|
||||
for artifact in artifacts:
|
||||
# Check if unchanged (idempotent ingestion)
|
||||
if self._is_unchanged(artifact, index):
|
||||
result.skipped += 1
|
||||
continue
|
||||
|
||||
# Enrich
|
||||
artifact = self._enrich_artifact(artifact)
|
||||
|
||||
# Store
|
||||
if not self.dry_run:
|
||||
self._store_artifact(artifact)
|
||||
|
||||
# Update index
|
||||
self._update_index(artifact, index)
|
||||
result.processed += 1
|
||||
|
||||
# Save index
|
||||
if not self.dry_run:
|
||||
index.save()
|
||||
|
||||
return result
|
||||
|
||||
def process_inbox(self) -> IngestionResult:
|
||||
"""Process all items in {context}/knowledge/inbox/.
|
||||
|
||||
Reads all markdown files from the inbox, enriches them,
|
||||
and moves them to the appropriate category folder.
|
||||
After processing, commits changes and creates tasks for action items.
|
||||
|
||||
Returns:
|
||||
IngestionResult with processing statistics.
|
||||
"""
|
||||
result = IngestionResult(context=self.context)
|
||||
inbox_path = self.knowledge_path / "inbox"
|
||||
|
||||
if not inbox_path.exists():
|
||||
return result
|
||||
|
||||
index = self._load_index()
|
||||
|
||||
# Load settings for git/task config
|
||||
settings = PipelineSettings()
|
||||
if self.sources_config_path.exists():
|
||||
try:
|
||||
_, settings = load_sources_config(self.sources_config_path)
|
||||
except Exception:
|
||||
pass # Use defaults if config load fails
|
||||
|
||||
# Track written paths for git commit
|
||||
written_paths: list[Path] = []
|
||||
|
||||
# Track action items for OrgMyLife
|
||||
all_action_items: list[tuple[str, list[ActionItem], str]] = []
|
||||
|
||||
for file_path in inbox_path.glob("*.md"):
|
||||
if file_path.name == "index.md":
|
||||
continue
|
||||
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
except Exception as e:
|
||||
result.errors.append(IngestError(
|
||||
source=str(file_path),
|
||||
error_type="read",
|
||||
message=str(e),
|
||||
))
|
||||
continue
|
||||
|
||||
# Create artifact from inbox file
|
||||
artifact = Artifact(
|
||||
metadata=ArtifactMetadata(
|
||||
type="inbox",
|
||||
title=file_path.stem,
|
||||
source_context=self.context,
|
||||
created=date.today(),
|
||||
),
|
||||
content=content,
|
||||
file_path=file_path,
|
||||
)
|
||||
|
||||
# Check if unchanged
|
||||
if self._is_unchanged(artifact, index):
|
||||
result.skipped += 1
|
||||
continue
|
||||
|
||||
# Enrich
|
||||
artifact = self._enrich_artifact(artifact)
|
||||
|
||||
# Collect action items for OrgMyLife
|
||||
enrichment_result = self.enrichment_agent.enrich(
|
||||
artifact.content, "inbox"
|
||||
)
|
||||
if enrichment_result.action_items:
|
||||
artifact_id = self._compute_artifact_id(artifact)
|
||||
all_action_items.append((
|
||||
artifact_id,
|
||||
enrichment_result.action_items,
|
||||
str(artifact.file_path),
|
||||
))
|
||||
|
||||
# Store in categorized folder
|
||||
if not self.dry_run:
|
||||
stored_path = self._store_artifact(artifact)
|
||||
written_paths.append(stored_path)
|
||||
|
||||
# Update index
|
||||
self._update_index(artifact, index)
|
||||
result.processed += 1
|
||||
|
||||
# Save index
|
||||
if not self.dry_run:
|
||||
index.save()
|
||||
if result.processed > 0:
|
||||
written_paths.append(self.index_path)
|
||||
|
||||
# Git auto-commit
|
||||
if not self.dry_run and written_paths:
|
||||
commit_sha = auto_commit(
|
||||
context=self.context,
|
||||
action="categorize",
|
||||
count=result.processed,
|
||||
source="inbox",
|
||||
paths=written_paths,
|
||||
no_commit=self.no_commit,
|
||||
auto_commit_enabled=settings.git.auto_commit,
|
||||
cwd=self.monorepo_root,
|
||||
)
|
||||
result.commit_sha = commit_sha
|
||||
|
||||
# OrgMyLife task creation for action items
|
||||
if not self.dry_run and settings.create_tasks and all_action_items:
|
||||
try:
|
||||
orgmylife = OrgMyLifeIntegration(
|
||||
context_path=self.knowledge_path,
|
||||
enabled=settings.create_tasks,
|
||||
)
|
||||
for artifact_id, action_items, source_path in all_action_items:
|
||||
mappings = orgmylife.create_tasks(
|
||||
artifact_id=artifact_id,
|
||||
action_items=action_items,
|
||||
source_path=source_path,
|
||||
)
|
||||
result.tasks_created += len(mappings)
|
||||
except Exception as e:
|
||||
logger.warning(f"OrgMyLife task creation failed: {e}")
|
||||
result.errors.append(IngestError(
|
||||
source="orgmylife",
|
||||
error_type="integration",
|
||||
message=str(e),
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slugify_simple(name: str) -> str:
|
||||
"""Simple slugification for wiki-links.
|
||||
|
||||
Args:
|
||||
name: The name to slugify.
|
||||
|
||||
Returns:
|
||||
Lowercase, hyphen-separated string.
|
||||
"""
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
text = name.strip().lower()
|
||||
# German umlaut replacements
|
||||
replacements = {"ä": "ae", "ö": "oe", "ü": "ue", "ß": "ss"}
|
||||
for char, repl in replacements.items():
|
||||
text = text.replace(char, repl)
|
||||
# Remove diacritics
|
||||
text = unicodedata.normalize("NFD", text)
|
||||
text = "".join(c for c in text if unicodedata.category(c) != "Mn")
|
||||
# Replace non-alphanumeric with hyphens
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text)
|
||||
text = re.sub(r"-+", "-", text).strip("-")
|
||||
return text
|
||||
@@ -0,0 +1,180 @@
|
||||
"""ContextRouter – Routet Artefakte in den richtigen Wissenskontext.
|
||||
|
||||
Bestimmt anhand von Quellkonfiguration und Metadaten, in welchen
|
||||
Kontext (bahn, dhive, privat) ein Artefakt gehört und berechnet
|
||||
den Zielpfad im entsprechenden Knowledge-Folder.
|
||||
|
||||
Requirements: 1.4, 2.3, 3.2, 3.5, 9.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.ingestion.config import SourceConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_CONTEXTS = ("bahn", "dhive", "privat")
|
||||
|
||||
CATEGORY_FOLDER_MAP: dict[str, str] = {
|
||||
"meeting": "meetings",
|
||||
"decision": "decisions",
|
||||
"project": "projects",
|
||||
"reference": "references",
|
||||
"link": "links",
|
||||
"inbox": "inbox",
|
||||
}
|
||||
|
||||
DEFAULT_FOLDER = "inbox"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Model
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class RoutingDecision:
|
||||
"""Result of context routing."""
|
||||
|
||||
context: str # bahn, dhive, privat
|
||||
target_folder: str # knowledge subfolder (meetings, decisions, etc.)
|
||||
artifact_path: Path # full path for the artifact
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def category_to_folder(category: str) -> str:
|
||||
"""Map a category name to the corresponding knowledge subfolder.
|
||||
|
||||
Known categories are mapped to their plural folder name.
|
||||
Unknown categories default to 'inbox'.
|
||||
|
||||
Args:
|
||||
category: The category string (e.g. 'meeting', 'decision').
|
||||
|
||||
Returns:
|
||||
The folder name (e.g. 'meetings', 'decisions', 'inbox').
|
||||
"""
|
||||
return CATEGORY_FOLDER_MAP.get(category.lower(), DEFAULT_FOLDER)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ContextRouter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ContextRouter:
|
||||
"""Routes incoming data to the correct context knowledge folder.
|
||||
|
||||
Uses source configuration or working directory to determine the
|
||||
target context, then computes the full artifact path within the
|
||||
monorepo structure.
|
||||
"""
|
||||
|
||||
def __init__(self, monorepo_root: Path) -> None:
|
||||
"""Initialize with the monorepo root path.
|
||||
|
||||
Args:
|
||||
monorepo_root: Absolute path to the monorepo root directory.
|
||||
"""
|
||||
self.monorepo_root = monorepo_root
|
||||
|
||||
def determine_context(
|
||||
self,
|
||||
source_config: SourceConfig | None = None,
|
||||
cwd: Path | None = None,
|
||||
) -> str:
|
||||
"""Determine target context from source config or working directory.
|
||||
|
||||
Resolution order:
|
||||
1. If source_config is provided and has a 'context' param, use that.
|
||||
2. Otherwise derive from cwd by checking if path contains a known
|
||||
context segment (/bahn/, /dhive/, /privat/).
|
||||
3. If neither works, raise ValueError.
|
||||
|
||||
Args:
|
||||
source_config: Optional source configuration with context param.
|
||||
cwd: Optional working directory to derive context from.
|
||||
|
||||
Returns:
|
||||
The context string ('bahn', 'dhive', or 'privat').
|
||||
|
||||
Raises:
|
||||
ValueError: If context cannot be determined from either source.
|
||||
"""
|
||||
# Strategy 1: Extract from source_config params
|
||||
if source_config is not None:
|
||||
context_from_config = source_config.params.get("context", "")
|
||||
if context_from_config and context_from_config in VALID_CONTEXTS:
|
||||
return context_from_config
|
||||
|
||||
# Strategy 2: Derive from working directory path
|
||||
if cwd is not None:
|
||||
# Normalize to forward slashes for consistent matching
|
||||
path_str = str(cwd).replace("\\", "/")
|
||||
for ctx in VALID_CONTEXTS:
|
||||
if f"/{ctx}/" in path_str or path_str.endswith(f"/{ctx}"):
|
||||
return ctx
|
||||
|
||||
raise ValueError(
|
||||
"Kontext konnte nicht bestimmt werden. "
|
||||
"Weder source_config.params['context'] noch cwd enthalten "
|
||||
f"einen gültigen Kontext ({', '.join(VALID_CONTEXTS)})."
|
||||
)
|
||||
|
||||
def resolve_target_path(
|
||||
self, context: str, category: str, filename: str
|
||||
) -> Path:
|
||||
"""Compute full path: {root}/{context}/knowledge/{category_folder}/{filename}.
|
||||
|
||||
Args:
|
||||
context: The target context ('bahn', 'dhive', or 'privat').
|
||||
category: The artifact category (e.g. 'meeting', 'decision').
|
||||
filename: The filename for the artifact.
|
||||
|
||||
Returns:
|
||||
The full Path for the artifact within the knowledge folder.
|
||||
"""
|
||||
folder = category_to_folder(category)
|
||||
return self.monorepo_root / context / "knowledge" / folder / filename
|
||||
|
||||
def route(
|
||||
self,
|
||||
category: str,
|
||||
filename: str,
|
||||
source_config: SourceConfig | None = None,
|
||||
cwd: Path | None = None,
|
||||
) -> RoutingDecision:
|
||||
"""Full routing: determine context and compute target path.
|
||||
|
||||
Convenience method combining determine_context and resolve_target_path.
|
||||
|
||||
Args:
|
||||
category: The artifact category.
|
||||
filename: The artifact filename.
|
||||
source_config: Optional source configuration.
|
||||
cwd: Optional working directory.
|
||||
|
||||
Returns:
|
||||
A RoutingDecision with context, target_folder, and artifact_path.
|
||||
|
||||
Raises:
|
||||
ValueError: If context cannot be determined.
|
||||
"""
|
||||
context = self.determine_context(source_config, cwd)
|
||||
folder = category_to_folder(category)
|
||||
artifact_path = self.resolve_target_path(context, category, filename)
|
||||
return RoutingDecision(
|
||||
context=context,
|
||||
target_folder=folder,
|
||||
artifact_path=artifact_path,
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Externe Integrationen für das Knowledge-Management.
|
||||
|
||||
Bindet externe Systeme wie OrgMyLife, Kiro und LiteLLM an die
|
||||
Ingestion-Pipeline an. Enthält außerdem den Federation-Sync-Filter
|
||||
für die Knowledge-Ordner-Behandlung bei der Team-Repo-Synchronisation.
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.integrations.federation import (
|
||||
FederationSyncFilter,
|
||||
FederationSyncFilterResult,
|
||||
SyncExclusion,
|
||||
)
|
||||
from monorepo.knowledge.integrations.kiro_client import (
|
||||
KiroClient,
|
||||
KiroEnrichmentResult,
|
||||
KiroUnavailableError,
|
||||
check_kiro_available,
|
||||
create_kiro_client,
|
||||
)
|
||||
from monorepo.knowledge.integrations.orgmylife import (
|
||||
OrgMyLifeIntegration,
|
||||
TaskMapping,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FederationSyncFilter",
|
||||
"FederationSyncFilterResult",
|
||||
"KiroClient",
|
||||
"KiroEnrichmentResult",
|
||||
"KiroUnavailableError",
|
||||
"OrgMyLifeIntegration",
|
||||
"SyncExclusion",
|
||||
"TaskMapping",
|
||||
"check_kiro_available",
|
||||
"create_kiro_client",
|
||||
]
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Federation-Sync-Filter für Knowledge-Artefakte.
|
||||
|
||||
Erweitert die bestehende Federation-Logik um Knowledge-Ordner-Behandlung:
|
||||
- Exkludiert Artefakte mit `shareable: false` aus der Sync-Liste
|
||||
- Exkludiert ctx-guard-markierte sensible Artefakte
|
||||
- Stellt sicher, dass keine kontextübergreifenden Links oder Referenzen
|
||||
auf andere Kontexte in das Team-Repo gelangen
|
||||
|
||||
Requirements: 14.1, 14.2, 14.3, 14.4, 14.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import parse_frontmatter, ArtifactMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Valid monorepo contexts
|
||||
VALID_CONTEXTS = {"bahn", "dhive", "privat", "shared"}
|
||||
|
||||
# Pattern for cross-context links in frontmatter/content
|
||||
# Matches references like "bahn/knowledge/...", "dhive/...", "privat/..."
|
||||
_CROSS_CONTEXT_LINK_PATTERN = re.compile(
|
||||
r"(?:^|\s|/|\"|\')(?:bahn|dhive|privat|shared)/",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# Pattern for wiki-links referencing other contexts: [[bahn/...]], [[dhive/...]]
|
||||
_WIKI_LINK_CROSS_CONTEXT = re.compile(
|
||||
r"\[\[(?:bahn|dhive|privat|shared)/[^\]]+\]\]",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncExclusion:
|
||||
"""Grund für den Ausschluss eines Artefakts von der Synchronisation."""
|
||||
|
||||
path: Path
|
||||
reason: str # "not_shareable", "ctx_guard_sensitive", "cross_context_links"
|
||||
details: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FederationSyncFilterResult:
|
||||
"""Ergebnis der Federation-Sync-Filterung."""
|
||||
|
||||
included: list[Path] = field(default_factory=list)
|
||||
excluded: list[SyncExclusion] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return len(self.included) + len(self.excluded)
|
||||
|
||||
|
||||
class FederationSyncFilter:
|
||||
"""Filtert Knowledge-Artefakte für die Federation-Synchronisation.
|
||||
|
||||
Stellt sicher, dass nur teilbare, nicht-sensible Artefakte ohne
|
||||
kontextübergreifende Referenzen in das Team-Repo synchronisiert werden.
|
||||
|
||||
Die Filterung respektiert die bestehende team-repos.yaml-Konfiguration
|
||||
und behandelt den Knowledge-Ordner als Teil des Kontextordners
|
||||
(keine separate Sync-Konfiguration nötig).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
monorepo_root: Path,
|
||||
context: str,
|
||||
ctx_guard: Any | None = None,
|
||||
) -> None:
|
||||
"""Initialisiert den Federation-Sync-Filter.
|
||||
|
||||
Args:
|
||||
monorepo_root: Wurzelverzeichnis des Monorepos.
|
||||
context: Arbeitskontext (bahn, dhive, privat).
|
||||
ctx_guard: Optionaler ContextGuard für Sensitivitätsprüfung.
|
||||
"""
|
||||
self.monorepo_root = monorepo_root
|
||||
self.context = context
|
||||
self.ctx_guard = ctx_guard
|
||||
self._other_contexts = VALID_CONTEXTS - {context}
|
||||
|
||||
def filter_knowledge_artifacts(
|
||||
self, knowledge_path: Path | None = None
|
||||
) -> FederationSyncFilterResult:
|
||||
"""Filtert alle Artefakte im Knowledge-Ordner für die Sync.
|
||||
|
||||
Scannt den Knowledge-Ordner des Kontexts und bestimmt für jedes
|
||||
Artefakt, ob es synchronisiert werden darf.
|
||||
|
||||
Args:
|
||||
knowledge_path: Optionaler Pfad zum Knowledge-Ordner.
|
||||
Standardmäßig: {monorepo_root}/{context}/knowledge/
|
||||
|
||||
Returns:
|
||||
FederationSyncFilterResult mit included/excluded Listen.
|
||||
"""
|
||||
if knowledge_path is None:
|
||||
knowledge_path = self.monorepo_root / self.context / "knowledge"
|
||||
|
||||
result = FederationSyncFilterResult()
|
||||
|
||||
if not knowledge_path.exists():
|
||||
return result
|
||||
|
||||
# Scan all markdown files in the knowledge folder
|
||||
for md_file in knowledge_path.rglob("*.md"):
|
||||
exclusion = self._check_artifact(md_file)
|
||||
if exclusion is not None:
|
||||
result.excluded.append(exclusion)
|
||||
logger.info(
|
||||
"Federation-Sync: Artefakt ausgeschlossen: %s (%s)",
|
||||
md_file.name,
|
||||
exclusion.reason,
|
||||
)
|
||||
else:
|
||||
result.included.append(md_file)
|
||||
|
||||
return result
|
||||
|
||||
def get_sync_file_list(
|
||||
self, knowledge_path: Path | None = None
|
||||
) -> list[Path]:
|
||||
"""Gibt die Liste der Dateien zurück, die synchronisiert werden dürfen.
|
||||
|
||||
Convenience-Methode, die nur die included-Pfade zurückgibt.
|
||||
|
||||
Args:
|
||||
knowledge_path: Optionaler Pfad zum Knowledge-Ordner.
|
||||
|
||||
Returns:
|
||||
Liste der Pfade, die in das Team-Repo synchronisiert werden dürfen.
|
||||
"""
|
||||
result = self.filter_knowledge_artifacts(knowledge_path)
|
||||
return result.included
|
||||
|
||||
def should_sync(self, file_path: Path) -> bool:
|
||||
"""Prüft ob eine einzelne Datei synchronisiert werden darf.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Artefakt-Datei.
|
||||
|
||||
Returns:
|
||||
True wenn das Artefakt synchronisiert werden darf.
|
||||
"""
|
||||
return self._check_artifact(file_path) is None
|
||||
|
||||
def _check_artifact(self, file_path: Path) -> SyncExclusion | None:
|
||||
"""Prüft ein einzelnes Artefakt auf Sync-Ausschluss.
|
||||
|
||||
Prüft in folgender Reihenfolge:
|
||||
1. shareable: false → ausschließen
|
||||
2. ctx-guard-sensibel → ausschließen
|
||||
3. Kontextübergreifende Links → ausschließen
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Markdown-Datei.
|
||||
|
||||
Returns:
|
||||
SyncExclusion wenn das Artefakt ausgeschlossen wird, sonst None.
|
||||
"""
|
||||
# Nur Markdown-Dateien prüfen
|
||||
if file_path.suffix.lower() not in (".md", ".markdown"):
|
||||
return None
|
||||
|
||||
# Skip non-artifact files like index.md without frontmatter
|
||||
try:
|
||||
artifact = parse_frontmatter(file_path)
|
||||
except (ValueError, FileNotFoundError, OSError):
|
||||
# Kein gültiges Frontmatter → als normales Markdown durchlassen
|
||||
return None
|
||||
|
||||
metadata = artifact.metadata
|
||||
|
||||
# 1. Check shareable field (Req 14.4)
|
||||
if not metadata.shareable:
|
||||
return SyncExclusion(
|
||||
path=file_path,
|
||||
reason="not_shareable",
|
||||
details=f"Artefakt '{metadata.title}' hat shareable: false",
|
||||
)
|
||||
|
||||
# 2. Check ctx-guard sensitivity (Req 14.5)
|
||||
if self._is_ctx_guard_sensitive(file_path):
|
||||
return SyncExclusion(
|
||||
path=file_path,
|
||||
reason="ctx_guard_sensitive",
|
||||
details=f"Artefakt '{metadata.title}' ist ctx-guard-geschützt",
|
||||
)
|
||||
|
||||
# 3. Check for cross-context links (Req 14.3)
|
||||
cross_ctx_issue = self._has_cross_context_references(
|
||||
artifact.content, metadata
|
||||
)
|
||||
if cross_ctx_issue:
|
||||
return SyncExclusion(
|
||||
path=file_path,
|
||||
reason="cross_context_links",
|
||||
details=cross_ctx_issue,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _is_ctx_guard_sensitive(self, file_path: Path) -> bool:
|
||||
"""Prüft ob eine Datei über ctx-guard als sensibel markiert ist.
|
||||
|
||||
Nutzt den bestehenden ContextGuard-Mechanismus: Wenn ein
|
||||
externer Kontext (z.B. "shared") keinen Zugriff auf die Datei hat,
|
||||
gilt sie als sensibel und wird von der Sync ausgeschlossen.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
|
||||
Returns:
|
||||
True wenn die Datei sensibel ist.
|
||||
"""
|
||||
if self.ctx_guard is None:
|
||||
return False
|
||||
|
||||
# A file is considered sensitive if the "shared" context cannot access it.
|
||||
# This uses the existing ContextGuard mechanism to determine sensitivity.
|
||||
try:
|
||||
return not self.ctx_guard.check_access("shared", file_path)
|
||||
except (FileNotFoundError, ValueError):
|
||||
# If ctx-guard can't determine access, assume safe
|
||||
return False
|
||||
|
||||
def _has_cross_context_references(
|
||||
self, content: str, metadata: ArtifactMetadata
|
||||
) -> str:
|
||||
"""Prüft ob ein Artefakt kontextübergreifende Referenzen enthält.
|
||||
|
||||
Untersucht sowohl den Content als auch die Frontmatter-Links
|
||||
auf Verweise auf andere Kontexte.
|
||||
|
||||
Args:
|
||||
content: Markdown-Inhalt des Artefakts.
|
||||
metadata: Geparste Metadaten.
|
||||
|
||||
Returns:
|
||||
Beschreibung des Problems oder leerer String wenn keine
|
||||
kontextübergreifenden Referenzen gefunden wurden.
|
||||
"""
|
||||
issues: list[str] = []
|
||||
|
||||
# Check frontmatter links for cross-context references
|
||||
for link in metadata.links:
|
||||
target = link.target
|
||||
for other_ctx in self._other_contexts:
|
||||
if target.startswith(f"{other_ctx}/") or f"/{other_ctx}/" in target:
|
||||
issues.append(
|
||||
f"Link-Ziel verweist auf Kontext '{other_ctx}': {target}"
|
||||
)
|
||||
|
||||
# Check people references for cross-context paths
|
||||
for person_ref in metadata.people:
|
||||
for other_ctx in self._other_contexts:
|
||||
if f"{other_ctx}/" in person_ref:
|
||||
issues.append(
|
||||
f"People-Referenz verweist auf Kontext '{other_ctx}': {person_ref}"
|
||||
)
|
||||
|
||||
# Check project references for cross-context paths
|
||||
for project_ref in metadata.projects:
|
||||
for other_ctx in self._other_contexts:
|
||||
if f"{other_ctx}/" in project_ref:
|
||||
issues.append(
|
||||
f"Project-Referenz verweist auf Kontext '{other_ctx}': {project_ref}"
|
||||
)
|
||||
|
||||
# Check content for wiki-links to other contexts
|
||||
for match in _WIKI_LINK_CROSS_CONTEXT.finditer(content):
|
||||
link_text = match.group(0)
|
||||
# Only flag if it references a different context
|
||||
for other_ctx in self._other_contexts:
|
||||
if f"[[{other_ctx}/" in link_text:
|
||||
issues.append(
|
||||
f"Wiki-Link verweist auf Kontext '{other_ctx}': {link_text}"
|
||||
)
|
||||
|
||||
if issues:
|
||||
return "; ".join(issues)
|
||||
|
||||
return ""
|
||||
@@ -0,0 +1,316 @@
|
||||
"""Kiro AI-Agent Client – LLM-Provider für Enrichment und OCR.
|
||||
|
||||
Stellt das Interface für den Kiro AI-Agent als primären LLM-Provider bereit.
|
||||
Da Kiro der laufende Orchestrator-Agent selbst ist, arbeitet die reale
|
||||
Invokation über die Agent-Runtime (kein REST-Call). Diese Klasse definiert
|
||||
das Interface, das der EnrichmentAgent aufruft, und implementiert die
|
||||
Fallback-Logik bei Nicht-Verfügbarkeit.
|
||||
|
||||
Nutzung:
|
||||
client = create_kiro_client()
|
||||
if client is not None:
|
||||
result = client.enrich_content("Meeting notes ...", "confluence")
|
||||
text = client.ocr_image(Path("scan.png"))
|
||||
else:
|
||||
# Kiro nicht verfügbar → enrichment_pending: true setzen
|
||||
...
|
||||
|
||||
Requirements: 9.3, 9.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class KiroUnavailableError(Exception):
|
||||
"""Raised when the Kiro AI-Agent is not available.
|
||||
|
||||
This indicates that:
|
||||
- The Kiro agent session is not active
|
||||
- The caller should set `enrichment_pending: true` on the artifact
|
||||
- The artifact can be re-enriched on a subsequent run when Kiro is available
|
||||
"""
|
||||
|
||||
def __init__(self, reason: str = "Kiro AI-Agent session is not active") -> None:
|
||||
self.reason = reason
|
||||
super().__init__(reason)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class KiroEnrichmentResult:
|
||||
"""Result from Kiro-based content enrichment.
|
||||
|
||||
Attributes:
|
||||
title: Generated or extracted title.
|
||||
category: Content classification (meeting, decision, project, reference, link, inbox).
|
||||
tags: Generated tags.
|
||||
people: Detected person names.
|
||||
projects: Detected project references.
|
||||
action_items: Extracted action item descriptions.
|
||||
confidence: Overall confidence score (0.0 - 1.0).
|
||||
"""
|
||||
|
||||
title: str = ""
|
||||
category: str = "inbox"
|
||||
tags: list[str] = field(default_factory=list)
|
||||
people: list[str] = field(default_factory=list)
|
||||
projects: list[str] = field(default_factory=list)
|
||||
action_items: list[str] = field(default_factory=list)
|
||||
confidence: float = 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol (Interface Definition)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class KiroClientProtocol(Protocol):
|
||||
"""Protocol defining the Kiro client interface.
|
||||
|
||||
Any implementation (real agent runtime, mock, or alternative provider)
|
||||
must satisfy this protocol for the EnrichmentAgent to use it.
|
||||
"""
|
||||
|
||||
def enrich_content(self, content: str, source_type: str = "") -> KiroEnrichmentResult:
|
||||
"""Analyze content and return enrichment metadata.
|
||||
|
||||
Args:
|
||||
content: The text content to analyze.
|
||||
source_type: Optional hint about content origin (e.g., "confluence", "email").
|
||||
|
||||
Returns:
|
||||
KiroEnrichmentResult with extracted metadata.
|
||||
|
||||
Raises:
|
||||
KiroUnavailableError: If the Kiro agent is not reachable.
|
||||
"""
|
||||
...
|
||||
|
||||
def ocr_image(self, image_path: Path) -> str:
|
||||
"""Extract text from an image using Kiro Vision.
|
||||
|
||||
Args:
|
||||
image_path: Path to the image file (PNG, JPG, JPEG).
|
||||
|
||||
Returns:
|
||||
Extracted text content from the image.
|
||||
|
||||
Raises:
|
||||
KiroUnavailableError: If the Kiro agent is not reachable.
|
||||
FileNotFoundError: If the image file does not exist.
|
||||
"""
|
||||
...
|
||||
|
||||
@property
|
||||
def is_available(self) -> bool:
|
||||
"""Check if the Kiro agent is currently available.
|
||||
|
||||
Returns:
|
||||
True if the agent session is active and ready.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class KiroClient:
|
||||
"""Wrapper for the Kiro AI-Agent as LLM provider.
|
||||
|
||||
Since Kiro IS the running agent itself, the actual LLM invocations happen
|
||||
through the agent runtime (not via REST API). This class defines the
|
||||
interface and implements availability checking + fallback signaling.
|
||||
|
||||
In the current implementation, Kiro availability is determined by an
|
||||
environment marker or explicit flag. When Kiro is unavailable (e.g., in
|
||||
CI pipelines or batch processing without an active agent session), all
|
||||
methods raise KiroUnavailableError so the caller can set
|
||||
`enrichment_pending: true`.
|
||||
|
||||
Usage in the EnrichmentAgent:
|
||||
client = create_kiro_client()
|
||||
if client is not None:
|
||||
try:
|
||||
result = client.enrich_content(text)
|
||||
except KiroUnavailableError:
|
||||
# Set enrichment_pending: true
|
||||
pass
|
||||
"""
|
||||
|
||||
def __init__(self, *, force_available: bool | None = None) -> None:
|
||||
"""Initialize the Kiro client.
|
||||
|
||||
Args:
|
||||
force_available: Override availability detection.
|
||||
- None: auto-detect from environment (default)
|
||||
- True: force client to report as available (for testing)
|
||||
- False: force client to report as unavailable
|
||||
"""
|
||||
self._force_available = force_available
|
||||
|
||||
@property
|
||||
def is_available(self) -> bool:
|
||||
"""Check if the Kiro agent is currently available.
|
||||
|
||||
Detection strategy:
|
||||
1. If force_available is set explicitly, use that value.
|
||||
2. Otherwise, check for KIRO_SESSION_ACTIVE env var.
|
||||
3. Default: assume NOT available (safe fallback that triggers
|
||||
enrichment_pending rather than silently skipping enrichment).
|
||||
|
||||
Returns:
|
||||
True if the agent session is active.
|
||||
"""
|
||||
if self._force_available is not None:
|
||||
return self._force_available
|
||||
|
||||
import os
|
||||
|
||||
# Check environment marker set by the Kiro agent runtime
|
||||
session_marker = os.environ.get("KIRO_SESSION_ACTIVE", "").lower()
|
||||
return session_marker in ("1", "true", "yes")
|
||||
|
||||
def enrich_content(self, content: str, source_type: str = "") -> KiroEnrichmentResult:
|
||||
"""Analyze content using the Kiro AI-Agent for enrichment.
|
||||
|
||||
When the agent is available, this method delegates to the Kiro runtime
|
||||
for LLM-based analysis. When unavailable, it raises KiroUnavailableError
|
||||
to signal that the caller should set enrichment_pending: true.
|
||||
|
||||
Args:
|
||||
content: The text content to analyze.
|
||||
source_type: Optional source type hint (e.g., "confluence", "jira").
|
||||
|
||||
Returns:
|
||||
KiroEnrichmentResult with LLM-generated metadata.
|
||||
|
||||
Raises:
|
||||
KiroUnavailableError: If the Kiro agent session is not active.
|
||||
"""
|
||||
if not self.is_available:
|
||||
raise KiroUnavailableError(
|
||||
"Kiro AI-Agent session is not active. "
|
||||
"Set KIRO_SESSION_ACTIVE=1 or run within an active Kiro session."
|
||||
)
|
||||
|
||||
# --- Agent Runtime Invocation Placeholder ---
|
||||
# In a live Kiro session, this is where the agent runtime would be
|
||||
# called to perform LLM-based content analysis. The agent processes
|
||||
# the content and returns structured enrichment data.
|
||||
#
|
||||
# For now, this returns a minimal placeholder result indicating that
|
||||
# the interface is working but the actual LLM call needs the agent
|
||||
# runtime context (which is available during interactive sessions).
|
||||
logger.info(
|
||||
"Kiro enrich_content called (content_length=%d, source_type=%s)",
|
||||
len(content),
|
||||
source_type or "unspecified",
|
||||
)
|
||||
|
||||
return KiroEnrichmentResult(
|
||||
title="",
|
||||
category="inbox",
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
def ocr_image(self, image_path: Path) -> str:
|
||||
"""Extract text from an image using Kiro Vision.
|
||||
|
||||
Sends the image to the Kiro agent which uses its vision capabilities
|
||||
to extract text content. This is the primary OCR method - no external
|
||||
OCR engine (Tesseract) is required when Kiro is active.
|
||||
|
||||
Args:
|
||||
image_path: Path to the image file.
|
||||
|
||||
Returns:
|
||||
Extracted text from the image.
|
||||
|
||||
Raises:
|
||||
KiroUnavailableError: If the Kiro agent is not active.
|
||||
FileNotFoundError: If the image file does not exist.
|
||||
"""
|
||||
if not image_path.exists():
|
||||
raise FileNotFoundError(f"Image file not found: {image_path}")
|
||||
|
||||
if not self.is_available:
|
||||
raise KiroUnavailableError(
|
||||
"Kiro Vision is not available. "
|
||||
"OCR requires an active Kiro agent session."
|
||||
)
|
||||
|
||||
# --- Agent Runtime Invocation Placeholder ---
|
||||
# In a live Kiro session, the image would be sent to the agent's
|
||||
# vision model for text extraction. The agent reads the image and
|
||||
# returns the extracted text.
|
||||
logger.info("Kiro ocr_image called (path=%s)", image_path)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Factory Function
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def create_kiro_client(*, force_available: bool | None = None) -> KiroClient | None:
|
||||
"""Factory function to create a KiroClient if the agent is available.
|
||||
|
||||
This is the recommended way to obtain a KiroClient instance. It checks
|
||||
availability and returns None if Kiro is not active, allowing the caller
|
||||
to handle the fallback gracefully.
|
||||
|
||||
Usage:
|
||||
client = create_kiro_client()
|
||||
if client is None:
|
||||
# Kiro not available → set enrichment_pending: true
|
||||
artifact.enrichment_pending = True
|
||||
else:
|
||||
result = client.enrich_content(content)
|
||||
|
||||
Args:
|
||||
force_available: Override auto-detection (for testing).
|
||||
|
||||
Returns:
|
||||
KiroClient instance if available, None otherwise.
|
||||
"""
|
||||
client = KiroClient(force_available=force_available)
|
||||
|
||||
if not client.is_available:
|
||||
logger.debug("Kiro client not available – enrichment will be marked as pending")
|
||||
return None
|
||||
|
||||
logger.debug("Kiro client is available")
|
||||
return client
|
||||
|
||||
|
||||
def check_kiro_available() -> bool:
|
||||
"""Quick check if Kiro is available without creating a client.
|
||||
|
||||
Useful for conditional logic where you just need to know if enrichment
|
||||
will work, without creating the full client object.
|
||||
|
||||
Returns:
|
||||
True if Kiro agent session is active.
|
||||
"""
|
||||
client = KiroClient()
|
||||
return client.is_available
|
||||
@@ -0,0 +1,326 @@
|
||||
"""OrgMyLife-Integration für das Knowledge-Management.
|
||||
|
||||
Erstellt Tasks in OrgMyLife aus erkannten Action-Items und verwaltet
|
||||
die Deduplication über eine Hash-basierte Mapping-Datei.
|
||||
|
||||
Requirements: 10.1, 10.2, 10.3, 10.4, 10.5, 10.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.ingestion.enrichment import ActionItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data Classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskMapping:
|
||||
"""Maps artifact action items to OrgMyLife task IDs.
|
||||
|
||||
Attributes:
|
||||
artifact_id: Identifier of the source artifact.
|
||||
action_item_hash: SHA-256 hash of the action item description.
|
||||
task_id: OrgMyLife task ID returned by the API.
|
||||
created: ISO date string when the mapping was created.
|
||||
"""
|
||||
|
||||
artifact_id: str
|
||||
action_item_hash: str
|
||||
task_id: int
|
||||
created: str # ISO date
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Default filename for the task mapping file.
|
||||
MAPPING_FILENAME = ".task-mapping.yaml"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OrgMyLife Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class OrgMyLifeIntegration:
|
||||
"""Creates tasks in OrgMyLife from detected action items.
|
||||
|
||||
Uses a mapping file (`{context}/knowledge/.task-mapping.yaml`) to track
|
||||
which action items have already been created as tasks, preventing
|
||||
duplicate task creation on repeated ingestion runs.
|
||||
|
||||
The actual OrgMyLife API call is stubbed with a TODO for now.
|
||||
The integration handles API failures gracefully by logging warnings
|
||||
and continuing without task creation.
|
||||
|
||||
Attributes:
|
||||
context_path: Path to the context knowledge folder.
|
||||
enabled: Whether task creation is enabled.
|
||||
"""
|
||||
|
||||
def __init__(self, context_path: Path, enabled: bool = True) -> None:
|
||||
"""Initialize the OrgMyLife integration.
|
||||
|
||||
Args:
|
||||
context_path: Path to the context knowledge folder
|
||||
(e.g., `{root}/bahn/knowledge/`).
|
||||
enabled: Whether task creation is active. If False,
|
||||
create_tasks() returns an empty list.
|
||||
"""
|
||||
self.context_path = context_path
|
||||
self.enabled = enabled
|
||||
self._mapping_path = context_path / MAPPING_FILENAME
|
||||
|
||||
def create_tasks(
|
||||
self,
|
||||
artifact_id: str,
|
||||
action_items: list[ActionItem],
|
||||
source_path: str,
|
||||
) -> list[TaskMapping]:
|
||||
"""Create OrgMyLife tasks for new action items (dedup via mapping file).
|
||||
|
||||
Only creates tasks for action items whose hash is not yet present
|
||||
in the mapping file. If the OrgMyLife API is unavailable, logs a
|
||||
warning and returns an empty list.
|
||||
|
||||
Args:
|
||||
artifact_id: Unique identifier of the source artifact.
|
||||
action_items: List of ActionItem instances to create tasks for.
|
||||
source_path: Path to the source artifact file (used as source_url).
|
||||
|
||||
Returns:
|
||||
List of TaskMapping instances for newly created tasks.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return []
|
||||
|
||||
if not action_items:
|
||||
return []
|
||||
|
||||
# Load existing mappings for deduplication
|
||||
existing_mappings = self._load_mapping()
|
||||
|
||||
new_mappings: list[TaskMapping] = []
|
||||
|
||||
for item in action_items:
|
||||
# Build dedup key: artifact_id + action_item_hash
|
||||
dedup_key = f"{artifact_id}:{item.hash}"
|
||||
|
||||
# Skip if already mapped (Requirement 10.5, 10.6)
|
||||
if dedup_key in existing_mappings:
|
||||
logger.debug(
|
||||
"Action item already mapped: %s (task_id=%d)",
|
||||
dedup_key,
|
||||
existing_mappings[dedup_key].task_id,
|
||||
)
|
||||
continue
|
||||
|
||||
# Create task via OrgMyLife API
|
||||
task_id = self._create_orgmylife_task(
|
||||
title=item.description,
|
||||
source_url=source_path,
|
||||
deadline=item.deadline,
|
||||
)
|
||||
|
||||
if task_id is None:
|
||||
# API failure – logged in _create_orgmylife_task, continue
|
||||
continue
|
||||
|
||||
# Record new mapping
|
||||
mapping = TaskMapping(
|
||||
artifact_id=artifact_id,
|
||||
action_item_hash=item.hash,
|
||||
task_id=task_id,
|
||||
created=date.today().isoformat(),
|
||||
)
|
||||
new_mappings.append(mapping)
|
||||
existing_mappings[dedup_key] = mapping
|
||||
|
||||
# Persist updated mappings
|
||||
if new_mappings:
|
||||
self._save_mapping(existing_mappings)
|
||||
logger.info(
|
||||
"Created %d OrgMyLife task(s) for artifact '%s'",
|
||||
len(new_mappings),
|
||||
artifact_id,
|
||||
)
|
||||
|
||||
return new_mappings
|
||||
|
||||
def _load_mapping(self) -> dict[str, TaskMapping]:
|
||||
"""Load task mappings from the YAML file.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping dedup keys (artifact_id:hash) to TaskMapping.
|
||||
"""
|
||||
if not self._mapping_path.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
raw = yaml.safe_load(self._mapping_path.read_text(encoding="utf-8"))
|
||||
except (yaml.YAMLError, OSError) as exc:
|
||||
logger.warning(
|
||||
"Failed to load task mapping from %s: %s",
|
||||
self._mapping_path,
|
||||
exc,
|
||||
)
|
||||
return {}
|
||||
|
||||
if not raw or not isinstance(raw, dict):
|
||||
return {}
|
||||
|
||||
mappings: dict[str, TaskMapping] = {}
|
||||
entries = raw.get("mappings", [])
|
||||
|
||||
if not isinstance(entries, list):
|
||||
return {}
|
||||
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
try:
|
||||
mapping = TaskMapping(
|
||||
artifact_id=str(entry["artifact_id"]),
|
||||
action_item_hash=str(entry["action_item_hash"]),
|
||||
task_id=int(entry["task_id"]),
|
||||
created=str(entry.get("created", "")),
|
||||
)
|
||||
dedup_key = f"{mapping.artifact_id}:{mapping.action_item_hash}"
|
||||
mappings[dedup_key] = mapping
|
||||
except (KeyError, ValueError, TypeError) as exc:
|
||||
logger.warning("Skipping invalid mapping entry: %s", exc)
|
||||
continue
|
||||
|
||||
return mappings
|
||||
|
||||
def _save_mapping(self, mappings: dict[str, TaskMapping]) -> None:
|
||||
"""Save task mappings to the YAML file.
|
||||
|
||||
Args:
|
||||
mappings: Dictionary of dedup keys to TaskMapping instances.
|
||||
"""
|
||||
entries: list[dict[str, Any]] = []
|
||||
for mapping in mappings.values():
|
||||
entries.append({
|
||||
"artifact_id": mapping.artifact_id,
|
||||
"action_item_hash": mapping.action_item_hash,
|
||||
"task_id": mapping.task_id,
|
||||
"created": mapping.created,
|
||||
})
|
||||
|
||||
data = {
|
||||
"version": "1.0",
|
||||
"mappings": entries,
|
||||
}
|
||||
|
||||
try:
|
||||
self._mapping_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._mapping_path.write_text(
|
||||
yaml.dump(data, default_flow_style=False, allow_unicode=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to save task mapping to %s: %s",
|
||||
self._mapping_path,
|
||||
exc,
|
||||
)
|
||||
|
||||
def _create_orgmylife_task(
|
||||
self,
|
||||
title: str,
|
||||
source_url: str,
|
||||
deadline: str | None = None,
|
||||
) -> int | None:
|
||||
"""Create a task in OrgMyLife via the REST API.
|
||||
|
||||
This method handles API errors gracefully: on failure it logs a
|
||||
warning and returns None (Requirement 10.4).
|
||||
|
||||
Args:
|
||||
title: Task title (action item description).
|
||||
source_url: Reference to the source artifact.
|
||||
deadline: Optional ISO date string for the task deadline.
|
||||
|
||||
Returns:
|
||||
The OrgMyLife task ID on success, or None on failure.
|
||||
"""
|
||||
# TODO: Implement actual OrgMyLife REST API call.
|
||||
# The OrgMyLife API endpoint for task creation:
|
||||
# POST /api/tasks
|
||||
# Body: {"title": title, "repo": "", "deadline": deadline}
|
||||
# Response: {"id": <int>, "title": ..., ...}
|
||||
#
|
||||
# For now, use a stub that simulates a successful API call.
|
||||
# In production, replace with actual HTTP client (httpx/requests).
|
||||
|
||||
try:
|
||||
task_id = self._call_orgmylife_api(title, source_url, deadline)
|
||||
return task_id
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Graceful error handling (Requirement 10.4)
|
||||
logger.warning(
|
||||
"OrgMyLife API error creating task '%s': %s",
|
||||
title,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
def _call_orgmylife_api(
|
||||
self,
|
||||
title: str,
|
||||
source_url: str,
|
||||
deadline: str | None = None,
|
||||
) -> int:
|
||||
"""Actually call the OrgMyLife API.
|
||||
|
||||
This is separated from _create_orgmylife_task to allow easy
|
||||
mocking in tests and future replacement with real HTTP calls.
|
||||
|
||||
Args:
|
||||
title: Task title.
|
||||
source_url: Reference to source artifact.
|
||||
deadline: Optional deadline ISO date string.
|
||||
|
||||
Returns:
|
||||
The created task ID.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the API call fails.
|
||||
"""
|
||||
# TODO: Replace with actual HTTP call to OrgMyLife API.
|
||||
# Example implementation with httpx:
|
||||
#
|
||||
# import httpx
|
||||
# response = httpx.post(
|
||||
# f"{ORGMYLIFE_BASE_URL}/api/tasks",
|
||||
# json={
|
||||
# "title": title,
|
||||
# "repo": "",
|
||||
# "source_url": source_url,
|
||||
# "deadline": deadline,
|
||||
# },
|
||||
# timeout=10.0,
|
||||
# )
|
||||
# response.raise_for_status()
|
||||
# return response.json()["id"]
|
||||
|
||||
# Stub: raise to signal that real API is not implemented yet.
|
||||
# In tests, this method is patched to return a task ID.
|
||||
raise NotImplementedError(
|
||||
"OrgMyLife API not yet connected. "
|
||||
"Patch _call_orgmylife_api in tests or implement HTTP client."
|
||||
)
|
||||
@@ -4,8 +4,32 @@ Jede Quellstrategie implementiert das Extrahieren von Wissensartefakten
|
||||
aus einem bestimmten Format/Medium (Markdown, Confluence, PDF, etc.).
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
from monorepo.knowledge.sources.chat import ChatSource
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource
|
||||
from monorepo.knowledge.sources.email import EmailSource
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.link import LinkRegistry
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.knowledge.sources.pdf import PDFSource
|
||||
from monorepo.knowledge.sources.wissensdatenbank import WissensdatenbankSource
|
||||
|
||||
__all__ = [
|
||||
"ChatSource",
|
||||
"ConfluenceSource",
|
||||
"EmailSource",
|
||||
"ExtractionResult",
|
||||
"JiraSource",
|
||||
"LinkRegistry",
|
||||
"MarkdownSource",
|
||||
"PDFSource",
|
||||
"SourceConfig",
|
||||
"SourceError",
|
||||
"SourceStrategy",
|
||||
"WissensdatenbankSource",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Abstrakte Basisklasse für Quellstrategien der ETL-Pipeline.
|
||||
|
||||
Definiert das SourceStrategy-Interface, das alle konkreten Quellstrategien
|
||||
(Markdown, PDF, Confluence, Jira, etc.) implementieren müssen.
|
||||
|
||||
Requirements: 2.2, 2.4, 2.7, 4.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Format Detection (Requirement 4.7)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Mapping von unterstützten Dateiendungen (ohne Punkt, lowercase) auf Formattypen.
|
||||
EXTENSION_FORMAT_MAP: dict[str, str] = {
|
||||
"md": "markdown",
|
||||
"txt": "text",
|
||||
"pdf": "pdf",
|
||||
"png": "image",
|
||||
"jpg": "image",
|
||||
"jpeg": "image",
|
||||
"docx": "docx",
|
||||
}
|
||||
|
||||
#: Menge aller unterstützten Endungen (lowercase, ohne Punkt).
|
||||
SUPPORTED_EXTENSIONS: set[str] = set(EXTENSION_FORMAT_MAP.keys())
|
||||
|
||||
|
||||
def detect_format(file_path: str) -> str:
|
||||
"""Erkennt den Formattyp anhand der Dateiendung.
|
||||
|
||||
Unterstützte Zuordnungen:
|
||||
- .md → "markdown"
|
||||
- .txt → "text"
|
||||
- .pdf → "pdf"
|
||||
- .png, .jpg, .jpeg → "image"
|
||||
- .docx → "docx"
|
||||
|
||||
Args:
|
||||
file_path: Ein Dateipfad (kann relativ oder absolut sein).
|
||||
|
||||
Returns:
|
||||
Der erkannte Formattyp als String.
|
||||
|
||||
Raises:
|
||||
ValueError: Falls die Dateiendung nicht unterstützt wird.
|
||||
"""
|
||||
# Support both Unix and Windows paths
|
||||
# Use suffix extraction that handles both separators
|
||||
path_str = file_path.strip()
|
||||
if not path_str:
|
||||
raise ValueError("Empty file path")
|
||||
|
||||
# Extract extension: find the last '.' in the filename portion
|
||||
# Handle both forward and backslash separators
|
||||
filename = path_str.replace("\\", "/").rsplit("/", 1)[-1]
|
||||
if "." not in filename or filename.startswith("."):
|
||||
raise ValueError(
|
||||
f"Unsupported or missing file extension in: {file_path!r}"
|
||||
)
|
||||
|
||||
ext = filename.rsplit(".", 1)[-1].lower()
|
||||
|
||||
if ext in EXTENSION_FORMAT_MAP:
|
||||
return EXTENSION_FORMAT_MAP[ext]
|
||||
|
||||
raise ValueError(
|
||||
f"Unsupported file extension '.{ext}' in: {file_path!r}. "
|
||||
f"Supported: {', '.join('.' + e for e in sorted(SUPPORTED_EXTENSIONS))}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Konfiguration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceConfig:
|
||||
"""Konfiguration für eine einzelne Quelle in sources.yaml.
|
||||
|
||||
Attributes:
|
||||
type: Quelltyp (confluence, jira, email, file, link, wissensdatenbank).
|
||||
name: Menschenlesbare Bezeichnung der Quelle.
|
||||
params: Typspezifische Verbindungs-/Filterparameter.
|
||||
target_folder: Optionaler Override für den Zielordner.
|
||||
sync_frequency: Synchronisierungsfrequenz (on-push, hourly, daily, manual).
|
||||
enabled: Ob die Quelle aktiv ist.
|
||||
"""
|
||||
|
||||
type: str
|
||||
name: str
|
||||
params: dict[str, Any] = field(default_factory=dict)
|
||||
target_folder: str = ""
|
||||
sync_frequency: str = "manual"
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ergebnisse und Fehler
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceError:
|
||||
"""Strukturierter Fehler bei der Quellextraktion.
|
||||
|
||||
Attributes:
|
||||
source_name: Bezeichnung der betroffenen Quelle.
|
||||
error_type: Fehlerkategorie (connection, auth, parse, write, llm).
|
||||
message: Menschenlesbare Fehlerbeschreibung.
|
||||
timestamp: Zeitstempel im ISO-8601-Format.
|
||||
retry: Ob ein Wiederholungsversuch sinnvoll ist.
|
||||
artifact_path: Betroffener Artefaktpfad, falls zutreffend.
|
||||
"""
|
||||
|
||||
source_name: str
|
||||
error_type: str
|
||||
message: str
|
||||
timestamp: str
|
||||
retry: bool
|
||||
artifact_path: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionResult:
|
||||
"""Ergebnis einer Quellstrategie-Extraktion.
|
||||
|
||||
Attributes:
|
||||
artifacts: Erfolgreich extrahierte Artefakte.
|
||||
errors: Fehler, die bei der Extraktion aufgetreten sind.
|
||||
skipped: Anzahl übersprungener Elemente (z.B. unveränderte Dateien).
|
||||
"""
|
||||
|
||||
artifacts: list[Artifact] = field(default_factory=list)
|
||||
errors: list[SourceError] = field(default_factory=list)
|
||||
skipped: int = 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Abstrakte Basisklasse
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SourceStrategy(ABC):
|
||||
"""Abstrakte Basisklasse für alle Quellstrategien.
|
||||
|
||||
Jede konkrete Quellstrategie (Markdown, PDF, Confluence, etc.) muss
|
||||
dieses Interface implementieren, um in die Ingestion-Pipeline integriert
|
||||
werden zu können.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
|
||||
"""Extrahiert Rohartefakte aus der konfigurierten Quelle.
|
||||
|
||||
Args:
|
||||
config: Die Quellkonfiguration mit Verbindungsparametern.
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Gibt an, ob diese Strategie inkrementelle Verarbeitung unterstützt.
|
||||
|
||||
Bei inkrementeller Verarbeitung werden nur neue oder geänderte
|
||||
Inhalte verarbeitet (z.B. über Content-Hashes).
|
||||
|
||||
Returns:
|
||||
True wenn inkrementelle Verarbeitung unterstützt wird.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,365 @@
|
||||
"""Chat-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Verarbeitet Chat-Exporte in Text- oder JSON-Format und erstellt
|
||||
strukturierte Markdown-Artefakte mit Frontmatter.
|
||||
|
||||
Unterstützte Formate:
|
||||
- Plain-Text mit Timestamp-Patterns (z.B. "[2025-01-15 14:30] User: Message")
|
||||
- JSON-Arrays mit Nachrichtenobjekten (sender, timestamp, text)
|
||||
|
||||
Requirements: 7.3, 7.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["ChatSource"]
|
||||
|
||||
# Supported chat export file extensions
|
||||
_CHAT_EXTENSIONS = {".txt", ".json"}
|
||||
|
||||
# Patterns for detecting timestamped chat messages in plain text
|
||||
# Format: [YYYY-MM-DD HH:MM] or [YYYY-MM-DD HH:MM:SS] followed by sender: message
|
||||
_TIMESTAMP_PATTERNS = [
|
||||
# [2025-01-15 14:30] User: Message
|
||||
re.compile(
|
||||
r"^\[(\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}(?::\d{2})?)\]\s*(.+?):\s*(.+)$",
|
||||
re.MULTILINE,
|
||||
),
|
||||
# 2025-01-15 14:30 - User: Message (WhatsApp-style)
|
||||
re.compile(
|
||||
r"^(\d{4}-\d{2}-\d{2}[\s,]+\d{2}:\d{2}(?::\d{2})?)\s*-\s*(.+?):\s*(.+)$",
|
||||
re.MULTILINE,
|
||||
),
|
||||
# DD.MM.YYYY, HH:MM - User: Message (German WhatsApp)
|
||||
re.compile(
|
||||
r"^(\d{2}\.\d{2}\.\d{4},?\s*\d{2}:\d{2}(?::\d{2})?)\s*-\s*(.+?):\s*(.+)$",
|
||||
re.MULTILINE,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Creates a URL-friendly slug from text."""
|
||||
slug = text.lower().strip()
|
||||
slug = re.sub(r"[^a-z0-9äöüß\s-]", "", slug)
|
||||
slug = re.sub(r"[\s_]+", "-", slug)
|
||||
slug = re.sub(r"-+", "-", slug)
|
||||
slug = slug.strip("-")
|
||||
return slug[:80] if slug else "chat-export"
|
||||
|
||||
|
||||
def _compute_content_hash(content: str) -> str:
|
||||
"""Computes SHA-256 hash of content."""
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
def _parse_date_str(date_str: str) -> datetime | None:
|
||||
"""Attempts to parse various date formats into a datetime."""
|
||||
formats = [
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%dT%H:%M",
|
||||
"%d.%m.%Y, %H:%M:%S",
|
||||
"%d.%m.%Y, %H:%M",
|
||||
"%d.%m.%Y %H:%M:%S",
|
||||
"%d.%m.%Y %H:%M",
|
||||
]
|
||||
date_str = date_str.strip()
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(date_str, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatMessage:
|
||||
"""A single parsed chat message."""
|
||||
|
||||
timestamp: datetime | None
|
||||
sender: str
|
||||
text: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ChatSource – implementiert SourceStrategy ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ChatSource(SourceStrategy):
|
||||
"""Quellstrategie: Verarbeitet Chat-Exporte (Text/JSON) aus einem Verzeichnis.
|
||||
|
||||
Unterstützt:
|
||||
- Plain-Text-Exporte mit erkennbaren Timestamp-Patterns
|
||||
- JSON-Exporte als Array von Message-Objekten
|
||||
|
||||
Erstellt pro Exportdatei ein Markdown-Artefakt mit Konversationsinhalt
|
||||
und Frontmatter-Metadaten (Teilnehmer, Zeitraum, Quelltyp).
|
||||
"""
|
||||
|
||||
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
|
||||
"""Extrahiert Chat-Artefakte aus dem konfigurierten Verzeichnis.
|
||||
|
||||
Args:
|
||||
config: Quellkonfiguration mit params["directory"] als Pfad.
|
||||
context: Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und Fehlern.
|
||||
"""
|
||||
directory = self._resolve_directory(config)
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
|
||||
if not directory.exists():
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Verzeichnis existiert nicht: {directory}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
))
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
if not directory.is_dir():
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Pfad ist kein Verzeichnis: {directory}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
))
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
# Find all chat export files
|
||||
chat_files = sorted(
|
||||
f for f in directory.rglob("*")
|
||||
if f.is_file() and f.suffix.lower() in _CHAT_EXTENSIONS
|
||||
)
|
||||
|
||||
for file_path in chat_files:
|
||||
try:
|
||||
artifact = self._process_chat_file(file_path, config.name)
|
||||
if artifact is not None:
|
||||
artifacts.append(artifact)
|
||||
except Exception as e:
|
||||
logger.warning("Fehler beim Verarbeiten von %s: %s", file_path, e)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler bei {file_path.name}: {e}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
artifact_path=str(file_path),
|
||||
))
|
||||
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Chat-Exporte unterstützen keine inkrementelle Verarbeitung."""
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_directory(self, config: SourceConfig) -> Path:
|
||||
"""Resolves the source directory from config."""
|
||||
dir_param = config.params.get("directory", "")
|
||||
if dir_param:
|
||||
return Path(dir_param)
|
||||
raise ValueError(
|
||||
"ChatSource benötigt params['directory'] in der Konfiguration."
|
||||
)
|
||||
|
||||
def _process_chat_file(self, file_path: Path, source_name: str) -> Artifact | None:
|
||||
"""Processes a single chat export file."""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix == ".json":
|
||||
return self._process_json_chat(file_path, source_name)
|
||||
elif suffix == ".txt":
|
||||
return self._process_text_chat(file_path, source_name)
|
||||
return None
|
||||
|
||||
def _process_json_chat(self, file_path: Path, source_name: str) -> Artifact | None:
|
||||
"""Processes a JSON chat export.
|
||||
|
||||
Expected format: array of objects with keys:
|
||||
- sender (str): message author
|
||||
- timestamp (str): ISO-format or similar date string
|
||||
- text (str): message content
|
||||
"""
|
||||
raw_text = file_path.read_text(encoding="utf-8")
|
||||
data = json.loads(raw_text)
|
||||
|
||||
if not isinstance(data, list):
|
||||
logger.warning("JSON-Chat %s ist kein Array, überspringe.", file_path)
|
||||
return None
|
||||
|
||||
messages: list[ChatMessage] = []
|
||||
for entry in data:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
sender = str(entry.get("sender", entry.get("author", entry.get("from", ""))))
|
||||
text = str(entry.get("text", entry.get("message", entry.get("content", ""))))
|
||||
ts_str = str(entry.get("timestamp", entry.get("date", entry.get("time", ""))))
|
||||
timestamp = _parse_date_str(ts_str) if ts_str else None
|
||||
if text:
|
||||
messages.append(ChatMessage(timestamp=timestamp, sender=sender, text=text))
|
||||
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
return self._build_artifact(file_path, messages, source_name, "json")
|
||||
|
||||
def _process_text_chat(self, file_path: Path, source_name: str) -> Artifact | None:
|
||||
"""Processes a plain-text chat export with timestamp patterns."""
|
||||
raw_text = file_path.read_text(encoding="utf-8")
|
||||
|
||||
messages: list[ChatMessage] = []
|
||||
|
||||
# Try each pattern until one matches
|
||||
for pattern in _TIMESTAMP_PATTERNS:
|
||||
matches = pattern.findall(raw_text)
|
||||
if matches:
|
||||
for match in matches:
|
||||
ts_str, sender, text = match
|
||||
timestamp = _parse_date_str(ts_str)
|
||||
messages.append(ChatMessage(
|
||||
timestamp=timestamp,
|
||||
sender=sender.strip(),
|
||||
text=text.strip(),
|
||||
))
|
||||
break
|
||||
|
||||
# If no pattern matched, treat entire file as a single message block
|
||||
if not messages:
|
||||
# Still create an artifact from the raw content
|
||||
messages.append(ChatMessage(
|
||||
timestamp=None,
|
||||
sender="unknown",
|
||||
text=raw_text.strip(),
|
||||
))
|
||||
|
||||
return self._build_artifact(file_path, messages, source_name, "text")
|
||||
|
||||
def _build_artifact(
|
||||
self,
|
||||
file_path: Path,
|
||||
messages: list[ChatMessage],
|
||||
source_name: str,
|
||||
source_format: str,
|
||||
) -> Artifact:
|
||||
"""Builds a Markdown artifact from parsed chat messages."""
|
||||
# Collect participants and date range
|
||||
participants: set[str] = set()
|
||||
dates: list[datetime] = []
|
||||
|
||||
for msg in messages:
|
||||
if msg.sender and msg.sender != "unknown":
|
||||
participants.add(msg.sender)
|
||||
if msg.timestamp:
|
||||
dates.append(msg.timestamp)
|
||||
|
||||
date_start = min(dates).date() if dates else None
|
||||
date_end = max(dates).date() if dates else None
|
||||
|
||||
# Build Markdown content
|
||||
title = f"Chat: {file_path.stem}"
|
||||
content_lines: list[str] = []
|
||||
content_lines.append(f"# {title}\n")
|
||||
content_lines.append("")
|
||||
|
||||
if participants:
|
||||
content_lines.append(f"**Teilnehmer:** {', '.join(sorted(participants))} ")
|
||||
if date_start and date_end:
|
||||
if date_start == date_end:
|
||||
content_lines.append(f"**Datum:** {date_start.isoformat()} ")
|
||||
else:
|
||||
content_lines.append(
|
||||
f"**Zeitraum:** {date_start.isoformat()} bis {date_end.isoformat()} "
|
||||
)
|
||||
content_lines.append("")
|
||||
content_lines.append("---\n")
|
||||
content_lines.append("")
|
||||
|
||||
# Render messages
|
||||
for msg in messages:
|
||||
ts_prefix = ""
|
||||
if msg.timestamp:
|
||||
ts_prefix = f"[{msg.timestamp.strftime('%Y-%m-%d %H:%M')}] "
|
||||
sender_prefix = f"**{msg.sender}:** " if msg.sender and msg.sender != "unknown" else ""
|
||||
content_lines.append(f"{ts_prefix}{sender_prefix}{msg.text} ")
|
||||
|
||||
content = "\n".join(content_lines)
|
||||
|
||||
# Build source dict for frontmatter
|
||||
source_dict: dict[str, str] = {
|
||||
"type": "chat",
|
||||
"file": str(file_path),
|
||||
"format": source_format,
|
||||
}
|
||||
|
||||
# Date range for frontmatter
|
||||
date_range = ""
|
||||
if date_start and date_end:
|
||||
date_range = f"{date_start.isoformat()}/{date_end.isoformat()}"
|
||||
elif date_start:
|
||||
date_range = date_start.isoformat()
|
||||
|
||||
if date_range:
|
||||
source_dict["date_range"] = date_range
|
||||
|
||||
# Build metadata
|
||||
metadata = ArtifactMetadata(
|
||||
type="chat",
|
||||
title=title,
|
||||
tags=["chat"],
|
||||
source_context="",
|
||||
created=date_start,
|
||||
updated=date_end,
|
||||
shareable=False,
|
||||
source=source_dict,
|
||||
category="inbox",
|
||||
people=sorted(participants),
|
||||
content_hash=_compute_content_hash(content),
|
||||
)
|
||||
|
||||
# Generate filename
|
||||
date_prefix = date_start.isoformat() if date_start else "undated"
|
||||
slug = _slugify(file_path.stem)
|
||||
filename = f"{date_prefix}-{slug}.md"
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=Path(filename),
|
||||
)
|
||||
@@ -0,0 +1,686 @@
|
||||
"""Confluence-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Implementiert die Anbindung an die Confluence REST API (Cloud und Server):
|
||||
- Einzelne Seiten abrufen
|
||||
- Seitenbäume rekursiv traversieren
|
||||
- FAQ-Spaces importieren
|
||||
- Inkrementelle Updates via Versions-Nummer
|
||||
- Graceful Error Handling (skip source, log error)
|
||||
|
||||
Seiteninhalte werden als Markdown extrahiert und mit reichhaltigem
|
||||
Frontmatter angereichert (URL, Space, Titel, Last-Modified).
|
||||
|
||||
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from base64 import b64encode
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTML → Markdown Konvertierung (leichtgewichtig, kein externes Paket)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _html_to_markdown(html: str) -> str:
|
||||
"""Konvertiert einfaches HTML in Markdown.
|
||||
|
||||
Unterstützt die gängigsten Confluence-HTML-Elemente:
|
||||
- Überschriften (h1-h6)
|
||||
- Absätze, Zeilenumbrüche
|
||||
- Listen (ul/ol/li)
|
||||
- Fett/Kursiv
|
||||
- Links
|
||||
- Code-Blöcke und Inline-Code
|
||||
- Tabellen (vereinfacht)
|
||||
|
||||
Args:
|
||||
html: HTML-String aus Confluence Storage Format.
|
||||
|
||||
Returns:
|
||||
Markdown-String.
|
||||
"""
|
||||
if not html:
|
||||
return ""
|
||||
|
||||
text = html
|
||||
|
||||
# Entferne CDATA und XML-Deklarationen
|
||||
text = re.sub(r"<!\[CDATA\[.*?\]\]>", "", text, flags=re.DOTALL)
|
||||
|
||||
# Code-Blöcke zuerst (vor allgemeinem Tag-Stripping)
|
||||
text = re.sub(
|
||||
r"<ac:structured-macro[^>]*ac:name=\"code\"[^>]*>.*?"
|
||||
r"<ac:plain-text-body>(.*?)</ac:plain-text-body>.*?"
|
||||
r"</ac:structured-macro>",
|
||||
r"\n```\n\1\n```\n",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
|
||||
# Confluence-spezifische Macros entfernen
|
||||
text = re.sub(
|
||||
r"<ac:structured-macro[^>]*>.*?</ac:structured-macro>",
|
||||
"",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
text = re.sub(r"<ac:[^>]*/>", "", text)
|
||||
text = re.sub(r"<ac:[^>]*>.*?</ac:[^>]*>", "", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<ri:[^>]*/>", "", text)
|
||||
text = re.sub(r"<ri:[^>]*>.*?</ri:[^>]*>", "", text, flags=re.DOTALL)
|
||||
|
||||
# Überschriften
|
||||
for level in range(1, 7):
|
||||
prefix = "#" * level
|
||||
text = re.sub(
|
||||
rf"<h{level}[^>]*>(.*?)</h{level}>",
|
||||
rf"\n{prefix} \1\n",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
|
||||
# Links
|
||||
text = re.sub(
|
||||
r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>',
|
||||
r"[\2](\1)",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
|
||||
# Fett und Kursiv
|
||||
text = re.sub(r"<strong[^>]*>(.*?)</strong>", r"**\1**", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<b[^>]*>(.*?)</b>", r"**\1**", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<em[^>]*>(.*?)</em>", r"*\1*", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<i[^>]*>(.*?)</i>", r"*\1*", text, flags=re.DOTALL)
|
||||
|
||||
# Code (inline)
|
||||
text = re.sub(r"<code[^>]*>(.*?)</code>", r"`\1`", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<pre[^>]*>(.*?)</pre>", r"\n```\n\1\n```\n", text, flags=re.DOTALL)
|
||||
|
||||
# Listen
|
||||
text = re.sub(r"<ul[^>]*>", "\n", text)
|
||||
text = re.sub(r"</ul>", "\n", text)
|
||||
text = re.sub(r"<ol[^>]*>", "\n", text)
|
||||
text = re.sub(r"</ol>", "\n", text)
|
||||
text = re.sub(r"<li[^>]*>(.*?)</li>", r"- \1\n", text, flags=re.DOTALL)
|
||||
|
||||
# Absätze und Zeilenumbrüche
|
||||
text = re.sub(r"<br\s*/?>", "\n", text)
|
||||
text = re.sub(r"<p[^>]*>(.*?)</p>", r"\1\n\n", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<div[^>]*>(.*?)</div>", r"\1\n", text, flags=re.DOTALL)
|
||||
|
||||
# Tabellen (vereinfachtes Markdown)
|
||||
text = re.sub(r"<table[^>]*>", "\n", text)
|
||||
text = re.sub(r"</table>", "\n", text)
|
||||
text = re.sub(r"<thead[^>]*>|</thead>", "", text)
|
||||
text = re.sub(r"<tbody[^>]*>|</tbody>", "", text)
|
||||
text = re.sub(r"<tr[^>]*>(.*?)</tr>", r"|\1\n", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<th[^>]*>(.*?)</th>", r" \1 |", text, flags=re.DOTALL)
|
||||
text = re.sub(r"<td[^>]*>(.*?)</td>", r" \1 |", text, flags=re.DOTALL)
|
||||
|
||||
# Alle verbleibenden HTML-Tags entfernen
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
|
||||
# HTML-Entities dekodieren
|
||||
text = unescape(text)
|
||||
|
||||
# Mehrfache Leerzeilen reduzieren
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
|
||||
return text.strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Confluence API Client
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConfluencePage:
|
||||
"""Repräsentiert eine einzelne Confluence-Seite."""
|
||||
|
||||
page_id: str
|
||||
title: str
|
||||
space_key: str
|
||||
version: int
|
||||
last_modified: str
|
||||
content_html: str
|
||||
url: str
|
||||
attachments: list[dict[str, str]]
|
||||
|
||||
|
||||
class ConfluenceClient:
|
||||
"""Leichtgewichtiger Client für die Confluence REST API.
|
||||
|
||||
Verwendet urllib aus der Standardbibliothek (keine requests-Abhängigkeit).
|
||||
Unterstützt Token-basierte Authentifizierung (Bearer oder Basic mit API-Token).
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str, token: str, username: str = "") -> None:
|
||||
"""Initialisiert den Confluence-Client.
|
||||
|
||||
Args:
|
||||
base_url: Basis-URL der Confluence-Instanz (z.B. https://confluence.example.com).
|
||||
token: API-Token für die Authentifizierung.
|
||||
username: Benutzername für Basic-Auth (optional, für Cloud).
|
||||
"""
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.token = token
|
||||
self.username = username
|
||||
|
||||
def _build_auth_header(self) -> str:
|
||||
"""Erstellt den Authorization-Header.
|
||||
|
||||
Returns:
|
||||
Authorization-Header-Wert.
|
||||
"""
|
||||
if self.username:
|
||||
# Cloud: Basic Auth mit E-Mail + API-Token
|
||||
credentials = f"{self.username}:{self.token}"
|
||||
encoded = b64encode(credentials.encode()).decode()
|
||||
return f"Basic {encoded}"
|
||||
# Server/DC: Bearer Token
|
||||
return f"Bearer {self.token}"
|
||||
|
||||
def _request(self, path: str) -> dict[str, Any]:
|
||||
"""Führt einen GET-Request gegen die Confluence REST API durch.
|
||||
|
||||
Args:
|
||||
path: API-Pfad (relativ zu base_url).
|
||||
|
||||
Returns:
|
||||
JSON-Response als Dictionary.
|
||||
|
||||
Raises:
|
||||
urllib.error.URLError: Bei Netzwerkfehlern.
|
||||
urllib.error.HTTPError: Bei HTTP-Fehlern (401, 403, 404, etc.).
|
||||
"""
|
||||
url = f"{self.base_url}{path}"
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Authorization", self._build_auth_header())
|
||||
req.add_header("Accept", "application/json")
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
def get_page(self, page_id: str) -> ConfluencePage:
|
||||
"""Ruft eine einzelne Confluence-Seite ab.
|
||||
|
||||
Args:
|
||||
page_id: Die ID der Seite.
|
||||
|
||||
Returns:
|
||||
ConfluencePage mit Inhalt und Metadaten.
|
||||
"""
|
||||
path = (
|
||||
f"/rest/api/content/{page_id}"
|
||||
f"?expand=body.storage,version,space,metadata.labels"
|
||||
f"&expand=children.attachment"
|
||||
)
|
||||
data = self._request(path)
|
||||
return self._parse_page_response(data)
|
||||
|
||||
def get_child_pages(self, page_id: str) -> list[str]:
|
||||
"""Ruft die IDs aller direkten Kindseiten ab.
|
||||
|
||||
Args:
|
||||
page_id: Die ID der Elternseite.
|
||||
|
||||
Returns:
|
||||
Liste von Seiten-IDs.
|
||||
"""
|
||||
path = f"/rest/api/content/{page_id}/child/page?limit=200"
|
||||
data = self._request(path)
|
||||
results = data.get("results", [])
|
||||
return [str(r["id"]) for r in results]
|
||||
|
||||
def get_space_pages(self, space_key: str) -> list[str]:
|
||||
"""Ruft alle Seiten-IDs eines Spaces ab.
|
||||
|
||||
Args:
|
||||
space_key: Der Space-Key (z.B. "ACV2").
|
||||
|
||||
Returns:
|
||||
Liste von Seiten-IDs.
|
||||
"""
|
||||
page_ids: list[str] = []
|
||||
start = 0
|
||||
limit = 50
|
||||
|
||||
while True:
|
||||
path = (
|
||||
f"/rest/api/content"
|
||||
f"?spaceKey={space_key}&type=page"
|
||||
f"&start={start}&limit={limit}"
|
||||
)
|
||||
data = self._request(path)
|
||||
results = data.get("results", [])
|
||||
page_ids.extend(str(r["id"]) for r in results)
|
||||
|
||||
# Paginierung
|
||||
size = data.get("size", 0)
|
||||
if size < limit:
|
||||
break
|
||||
start += limit
|
||||
|
||||
return page_ids
|
||||
|
||||
def get_page_tree(self, root_page_id: str) -> list[str]:
|
||||
"""Traversiert einen Seitenbaum rekursiv und sammelt alle Seiten-IDs.
|
||||
|
||||
Args:
|
||||
root_page_id: Die ID der Wurzelseite.
|
||||
|
||||
Returns:
|
||||
Liste aller Seiten-IDs im Baum (inklusive Wurzel).
|
||||
"""
|
||||
collected: list[str] = [root_page_id]
|
||||
queue = [root_page_id]
|
||||
|
||||
while queue:
|
||||
current_id = queue.pop(0)
|
||||
children = self.get_child_pages(current_id)
|
||||
for child_id in children:
|
||||
if child_id not in collected:
|
||||
collected.append(child_id)
|
||||
queue.append(child_id)
|
||||
|
||||
return collected
|
||||
|
||||
def _parse_page_response(self, data: dict[str, Any]) -> ConfluencePage:
|
||||
"""Parst die API-Antwort in ein ConfluencePage-Objekt.
|
||||
|
||||
Args:
|
||||
data: JSON-Response der Confluence API.
|
||||
|
||||
Returns:
|
||||
ConfluencePage mit extrahierten Daten.
|
||||
"""
|
||||
page_id = str(data.get("id", ""))
|
||||
title = data.get("title", "Untitled")
|
||||
|
||||
# Space
|
||||
space = data.get("space", {})
|
||||
space_key = space.get("key", "")
|
||||
|
||||
# Version
|
||||
version_info = data.get("version", {})
|
||||
version = version_info.get("number", 1)
|
||||
last_modified = version_info.get("when", "")
|
||||
|
||||
# Content
|
||||
body = data.get("body", {})
|
||||
storage = body.get("storage", {})
|
||||
content_html = storage.get("value", "")
|
||||
|
||||
# URL
|
||||
base_url = self.base_url
|
||||
links = data.get("_links", {})
|
||||
web_ui = links.get("webui", f"/pages/viewpage.action?pageId={page_id}")
|
||||
url = f"{base_url}{web_ui}"
|
||||
|
||||
# Attachments
|
||||
attachments: list[dict[str, str]] = []
|
||||
children = data.get("children", {})
|
||||
attachment_data = children.get("attachment", {})
|
||||
for att in attachment_data.get("results", []):
|
||||
att_title = att.get("title", "")
|
||||
att_links = att.get("_links", {})
|
||||
att_download = att_links.get("download", "")
|
||||
if att_title:
|
||||
attachments.append({
|
||||
"title": att_title,
|
||||
"url": f"{base_url}{att_download}" if att_download else "",
|
||||
})
|
||||
|
||||
return ConfluencePage(
|
||||
page_id=page_id,
|
||||
title=title,
|
||||
space_key=space_key,
|
||||
version=version,
|
||||
last_modified=last_modified,
|
||||
content_html=content_html,
|
||||
url=url,
|
||||
attachments=attachments,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Confluence Source Strategy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ConfluenceSource(SourceStrategy):
|
||||
"""Quellstrategie: Confluence REST API.
|
||||
|
||||
Unterstützt drei Abrufmodi:
|
||||
1. Einzelne Seiten (params.pages: list[str] mit Page-IDs)
|
||||
2. Seitenbäume (params.tree_roots: list[str] mit Root-Page-IDs)
|
||||
3. Spaces (params.spaces: list[str] mit Space-Keys)
|
||||
|
||||
Inkrementelle Updates werden über die Confluence-Versionsnummer gesteuert.
|
||||
Bekannte Versionen können über set_known_versions() gesetzt werden.
|
||||
|
||||
Fehler bei der API-Kommunikation werden graceful behandelt:
|
||||
- Einzelne fehlgeschlagene Seiten werden übersprungen
|
||||
- Bei kompletten API-Ausfällen wird die gesamte Quelle übersprungen
|
||||
- Bestehende Artefakte werden niemals gelöscht
|
||||
|
||||
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialisiert die ConfluenceSource."""
|
||||
self._known_versions: dict[str, int] = {}
|
||||
|
||||
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
|
||||
"""Extrahiert Artefakte aus Confluence.
|
||||
|
||||
Args:
|
||||
config: SourceConfig mit Confluence-Verbindungsparametern.
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
skipped = 0
|
||||
|
||||
# Client initialisieren
|
||||
try:
|
||||
client = self._create_client(config)
|
||||
except ValueError as e:
|
||||
logger.error("Confluence-Konfigurationsfehler: %s", e)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="auth",
|
||||
message=str(e),
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=False,
|
||||
))
|
||||
return ExtractionResult(artifacts=[], errors=errors, skipped=0)
|
||||
|
||||
# Seiten-IDs sammeln
|
||||
page_ids = self._collect_page_ids(client, config, errors)
|
||||
|
||||
# Seiten abrufen und verarbeiten
|
||||
for page_id in page_ids:
|
||||
try:
|
||||
page = client.get_page(page_id)
|
||||
|
||||
# Inkrementelle Verarbeitung: Version-Check
|
||||
known_version = self._known_versions.get(page_id, 0)
|
||||
if known_version >= page.version:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
artifact = self._page_to_artifact(page, context)
|
||||
artifacts.append(artifact)
|
||||
|
||||
except (urllib.error.URLError, urllib.error.HTTPError) as e:
|
||||
logger.warning(
|
||||
"Seite %s konnte nicht abgerufen werden: %s", page_id, e
|
||||
)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="connection",
|
||||
message=f"Seite {page_id}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=True,
|
||||
artifact_path=f"confluence:{page_id}",
|
||||
))
|
||||
except (json.JSONDecodeError, KeyError, TypeError) as e:
|
||||
logger.warning(
|
||||
"Seite %s konnte nicht geparst werden: %s", page_id, e
|
||||
)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Seite {page_id}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=False,
|
||||
artifact_path=f"confluence:{page_id}",
|
||||
))
|
||||
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors, skipped=skipped)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Confluence unterstützt inkrementelle Verarbeitung via Versions-Nummer.
|
||||
|
||||
Returns:
|
||||
True – inkrementelle Verarbeitung wird unterstützt.
|
||||
"""
|
||||
return True
|
||||
|
||||
def set_known_versions(self, versions: dict[str, int]) -> None:
|
||||
"""Setzt bekannte Seitenversionen für inkrementelle Verarbeitung.
|
||||
|
||||
Args:
|
||||
versions: Mapping von Page-ID zu bekannter Versionsnummer.
|
||||
Seiten mit gleicher oder niedrigerer Version werden übersprungen.
|
||||
"""
|
||||
self._known_versions = dict(versions)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Hilfsmethoden
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _create_client(self, config: SourceConfig) -> ConfluenceClient:
|
||||
"""Erstellt einen ConfluenceClient aus der SourceConfig.
|
||||
|
||||
Args:
|
||||
config: SourceConfig mit Confluence-Parametern.
|
||||
|
||||
Returns:
|
||||
Konfigurierter ConfluenceClient.
|
||||
|
||||
Raises:
|
||||
ValueError: Bei fehlenden Pflichtparametern.
|
||||
"""
|
||||
params = config.params
|
||||
base_url = params.get("base_url", "")
|
||||
if not base_url:
|
||||
raise ValueError("Confluence base_url ist nicht konfiguriert")
|
||||
|
||||
# Token kann unter verschiedenen Keys liegen
|
||||
token = params.get("token", "") or params.get("api_key", "")
|
||||
if not token:
|
||||
raise ValueError(
|
||||
"Confluence Token fehlt (weder 'token' noch 'api_key' in params)"
|
||||
)
|
||||
|
||||
username = params.get("username", "")
|
||||
|
||||
return ConfluenceClient(base_url=base_url, token=token, username=username)
|
||||
|
||||
def _collect_page_ids(
|
||||
self,
|
||||
client: ConfluenceClient,
|
||||
config: SourceConfig,
|
||||
errors: list[SourceError],
|
||||
) -> list[str]:
|
||||
"""Sammelt alle zu verarbeitenden Seiten-IDs.
|
||||
|
||||
Kombiniert IDs aus pages, tree_roots und spaces.
|
||||
|
||||
Args:
|
||||
client: Der Confluence-Client.
|
||||
config: Die Quellkonfiguration.
|
||||
errors: Fehlerliste zum Ergänzen.
|
||||
|
||||
Returns:
|
||||
Deduplizierte Liste von Seiten-IDs.
|
||||
"""
|
||||
params = config.params
|
||||
page_ids: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def _add_ids(ids: list[str]) -> None:
|
||||
for pid in ids:
|
||||
if pid not in seen:
|
||||
seen.add(pid)
|
||||
page_ids.append(pid)
|
||||
|
||||
# 1. Einzelne Seiten
|
||||
pages = params.get("pages", [])
|
||||
if isinstance(pages, list):
|
||||
_add_ids([str(p) for p in pages])
|
||||
|
||||
# 2. Seitenbäume
|
||||
tree_roots = params.get("tree_roots", [])
|
||||
if isinstance(tree_roots, list):
|
||||
for root_id in tree_roots:
|
||||
try:
|
||||
tree_ids = client.get_page_tree(str(root_id))
|
||||
_add_ids(tree_ids)
|
||||
except (urllib.error.URLError, urllib.error.HTTPError) as e:
|
||||
logger.warning(
|
||||
"Seitenbaum %s nicht abrufbar: %s", root_id, e
|
||||
)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="connection",
|
||||
message=f"Seitenbaum {root_id}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=True,
|
||||
))
|
||||
|
||||
# 3. Spaces
|
||||
spaces = params.get("spaces", [])
|
||||
if isinstance(spaces, list):
|
||||
for space_key in spaces:
|
||||
try:
|
||||
space_ids = client.get_space_pages(str(space_key))
|
||||
_add_ids(space_ids)
|
||||
except (urllib.error.URLError, urllib.error.HTTPError) as e:
|
||||
logger.warning(
|
||||
"Space %s nicht abrufbar: %s", space_key, e
|
||||
)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="connection",
|
||||
message=f"Space {space_key}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=True,
|
||||
))
|
||||
|
||||
return page_ids
|
||||
|
||||
def _page_to_artifact(self, page: ConfluencePage, context: str) -> Artifact:
|
||||
"""Konvertiert eine ConfluencePage in ein Artifact.
|
||||
|
||||
Args:
|
||||
page: Die Confluence-Seite mit Inhalt und Metadaten.
|
||||
context: Der Zielkontext.
|
||||
|
||||
Returns:
|
||||
Artifact mit Markdown-Inhalt und angereichertem Frontmatter.
|
||||
"""
|
||||
# HTML → Markdown
|
||||
markdown_content = _html_to_markdown(page.content_html)
|
||||
|
||||
# Titel als Überschrift voranstellen
|
||||
full_content = f"# {page.title}\n\n{markdown_content}"
|
||||
|
||||
# Anhänge als Links anhängen
|
||||
if page.attachments:
|
||||
full_content += "\n\n## Anhänge\n\n"
|
||||
for att in page.attachments:
|
||||
att_title = att.get("title", "Anhang")
|
||||
att_url = att.get("url", "")
|
||||
if att_url:
|
||||
full_content += f"- [{att_title}]({att_url})\n"
|
||||
else:
|
||||
full_content += f"- {att_title}\n"
|
||||
|
||||
# Content-Hash berechnen
|
||||
content_hash = f"sha256:{hashlib.sha256(full_content.encode()).hexdigest()}"
|
||||
|
||||
# Last-Modified parsen
|
||||
created_date = date.today()
|
||||
if page.last_modified:
|
||||
try:
|
||||
# Confluence ISO Format: 2024-01-15T10:30:00.000+01:00
|
||||
dt = datetime.fromisoformat(
|
||||
page.last_modified.replace("Z", "+00:00")
|
||||
)
|
||||
created_date = dt.date()
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Slug für Dateinamen
|
||||
slug = self._slugify(page.title)
|
||||
file_path = Path(f"{context}/knowledge/references/{slug}.md")
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type="reference",
|
||||
title=page.title,
|
||||
tags=[f"space:{page.space_key}"],
|
||||
source_context=context,
|
||||
created=created_date,
|
||||
updated=created_date,
|
||||
shareable=True,
|
||||
content_hash=content_hash,
|
||||
source={
|
||||
"type": "confluence",
|
||||
"url": page.url,
|
||||
"space": page.space_key,
|
||||
"page_id": page.page_id,
|
||||
"version": str(page.version),
|
||||
},
|
||||
category="references",
|
||||
)
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=full_content,
|
||||
file_path=file_path,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _slugify(text: str) -> str:
|
||||
"""Erzeugt einen URL-freundlichen Slug aus einem Text.
|
||||
|
||||
Args:
|
||||
text: Eingabetext (z.B. Seitentitel).
|
||||
|
||||
Returns:
|
||||
Slug in lowercase mit Bindestrichen.
|
||||
"""
|
||||
slug = text.lower().strip()
|
||||
# Umlaute ersetzen
|
||||
replacements = {
|
||||
"ä": "ae", "ö": "oe", "ü": "ue", "ß": "ss",
|
||||
"Ä": "ae", "Ö": "oe", "Ü": "ue",
|
||||
}
|
||||
for char, replacement in replacements.items():
|
||||
slug = slug.replace(char, replacement)
|
||||
# Nicht-alphanumerische Zeichen durch Bindestriche ersetzen
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
||||
# Führende/nachstehende Bindestriche entfernen
|
||||
slug = slug.strip("-")
|
||||
return slug or "untitled"
|
||||
@@ -0,0 +1,433 @@
|
||||
"""Email-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Verarbeitet exportierte E-Mail-Dateien (.eml, .msg) und extrahiert
|
||||
Absender, Empfänger, Betreff, Datum und Body als strukturierte
|
||||
Markdown-Artefakte. Anhänge werden als separate Artefakte referenziert.
|
||||
|
||||
Requirements: 7.1, 7.2, 7.5, 7.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import email
|
||||
import email.utils
|
||||
import hashlib
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from email.message import Message
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["EmailSource"]
|
||||
|
||||
# Supported email file extensions
|
||||
_EMAIL_EXTENSIONS = {".eml", ".msg"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Creates a URL-friendly slug from text."""
|
||||
import re
|
||||
|
||||
slug = text.lower().strip()
|
||||
slug = re.sub(r"[^a-z0-9äöüß\s-]", "", slug)
|
||||
slug = re.sub(r"[\s_]+", "-", slug)
|
||||
slug = re.sub(r"-+", "-", slug)
|
||||
slug = slug.strip("-")
|
||||
return slug[:80] if slug else "untitled"
|
||||
|
||||
|
||||
def _compute_content_hash(content: str) -> str:
|
||||
"""Computes SHA-256 hash of content."""
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
def _parse_email_date(date_str: str | None) -> date | None:
|
||||
"""Parses an email date header into a date object."""
|
||||
if not date_str:
|
||||
return None
|
||||
try:
|
||||
parsed = email.utils.parsedate_to_datetime(date_str)
|
||||
return parsed.date()
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _extract_body(msg: Message) -> str:
|
||||
"""Extracts the body from an email message.
|
||||
|
||||
Prefers plain text, falls back to HTML with basic stripping.
|
||||
"""
|
||||
if msg.is_multipart():
|
||||
plain_parts: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
|
||||
for part in msg.walk():
|
||||
content_type = part.get_content_type()
|
||||
disposition = str(part.get("Content-Disposition", ""))
|
||||
|
||||
# Skip attachments
|
||||
if "attachment" in disposition:
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload is None:
|
||||
continue
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
text = payload.decode(charset, errors="replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
continue
|
||||
|
||||
if content_type == "text/plain":
|
||||
plain_parts.append(text)
|
||||
elif content_type == "text/html":
|
||||
html_parts.append(text)
|
||||
|
||||
if plain_parts:
|
||||
return "\n".join(plain_parts)
|
||||
if html_parts:
|
||||
return _strip_html("\n".join(html_parts))
|
||||
return ""
|
||||
else:
|
||||
# Not multipart
|
||||
try:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload is None:
|
||||
return ""
|
||||
charset = msg.get_content_charset() or "utf-8"
|
||||
text = payload.decode(charset, errors="replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
return ""
|
||||
|
||||
if msg.get_content_type() == "text/html":
|
||||
return _strip_html(text)
|
||||
return text
|
||||
|
||||
|
||||
def _strip_html(html: str) -> str:
|
||||
"""Basic HTML tag stripping for fallback body extraction."""
|
||||
import re
|
||||
|
||||
# Remove script/style blocks
|
||||
text = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
|
||||
# Replace <br> and <p> with newlines
|
||||
text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
|
||||
text = re.sub(r"</?p[^>]*>", "\n", text, flags=re.IGNORECASE)
|
||||
# Remove all remaining tags
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
# Collapse whitespace
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _extract_attachments(msg: Message) -> list[dict[str, str]]:
|
||||
"""Extracts attachment metadata from an email message.
|
||||
|
||||
Returns a list of dicts with 'filename' and 'content_type'.
|
||||
"""
|
||||
attachments: list[dict[str, str]] = []
|
||||
|
||||
if not msg.is_multipart():
|
||||
return attachments
|
||||
|
||||
for part in msg.walk():
|
||||
disposition = str(part.get("Content-Disposition", ""))
|
||||
if "attachment" in disposition:
|
||||
filename = part.get_filename() or "unnamed-attachment"
|
||||
content_type = part.get_content_type() or "application/octet-stream"
|
||||
attachments.append({
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
})
|
||||
|
||||
return attachments
|
||||
|
||||
|
||||
def _format_recipients(recipients_str: str | None) -> list[str]:
|
||||
"""Parses a recipient header into a list of email addresses/names."""
|
||||
if not recipients_str:
|
||||
return []
|
||||
# Parse multiple addresses
|
||||
addresses = email.utils.getaddresses([recipients_str])
|
||||
result: list[str] = []
|
||||
for name, addr in addresses:
|
||||
if name and addr:
|
||||
result.append(f"{name} <{addr}>")
|
||||
elif addr:
|
||||
result.append(addr)
|
||||
elif name:
|
||||
result.append(name)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EmailSource – implementiert SourceStrategy ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EmailSource(SourceStrategy):
|
||||
"""Quellstrategie: Verarbeitet E-Mail-Dateien (.eml/.msg) aus einem Verzeichnis.
|
||||
|
||||
Extrahiert Absender, Empfänger, Betreff, Datum und Body.
|
||||
Erstellt Markdown-Artefakte mit entsprechendem Frontmatter.
|
||||
Anhänge werden als Links im Artefakt-Body referenziert.
|
||||
|
||||
.msg-Dateien erfordern das optionale Paket `extract_msg`. Ist es nicht
|
||||
installiert, wird ein Fehler geloggt und die Datei übersprungen.
|
||||
"""
|
||||
|
||||
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
|
||||
"""Extrahiert E-Mail-Artefakte aus dem konfigurierten Verzeichnis.
|
||||
|
||||
Args:
|
||||
config: Quellkonfiguration mit params["directory"] als Pfad.
|
||||
context: Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und Fehlern.
|
||||
"""
|
||||
directory = self._resolve_directory(config)
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
|
||||
if not directory.exists():
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Verzeichnis existiert nicht: {directory}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
))
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
if not directory.is_dir():
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Pfad ist kein Verzeichnis: {directory}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
))
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
# Find all email files
|
||||
email_files = sorted(
|
||||
f for f in directory.rglob("*")
|
||||
if f.is_file() and f.suffix.lower() in _EMAIL_EXTENSIONS
|
||||
)
|
||||
|
||||
for file_path in email_files:
|
||||
try:
|
||||
artifact = self._process_email_file(file_path, config.name)
|
||||
if artifact is not None:
|
||||
artifacts.append(artifact)
|
||||
except Exception as e:
|
||||
logger.warning("Fehler beim Verarbeiten von %s: %s", file_path, e)
|
||||
errors.append(SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler bei {file_path.name}: {e}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
artifact_path=str(file_path),
|
||||
))
|
||||
|
||||
return ExtractionResult(artifacts=artifacts, errors=errors)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Email-Quellen unterstützen keine inkrementelle Verarbeitung."""
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_directory(self, config: SourceConfig) -> Path:
|
||||
"""Resolves the source directory from config."""
|
||||
dir_param = config.params.get("directory", "")
|
||||
if dir_param:
|
||||
return Path(dir_param)
|
||||
raise ValueError(
|
||||
"EmailSource benötigt params['directory'] in der Konfiguration."
|
||||
)
|
||||
|
||||
def _process_email_file(self, file_path: Path, source_name: str) -> Artifact | None:
|
||||
"""Processes a single email file and returns an Artifact."""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix == ".eml":
|
||||
return self._process_eml(file_path, source_name)
|
||||
elif suffix == ".msg":
|
||||
return self._process_msg(file_path, source_name)
|
||||
return None
|
||||
|
||||
def _process_eml(self, file_path: Path, source_name: str) -> Artifact:
|
||||
"""Processes a .eml file using Python's built-in email module."""
|
||||
raw_bytes = file_path.read_bytes()
|
||||
msg = email.message_from_bytes(raw_bytes)
|
||||
|
||||
# Extract fields
|
||||
sender = msg.get("From", "")
|
||||
to_raw = msg.get("To", "")
|
||||
cc_raw = msg.get("Cc", "")
|
||||
subject = msg.get("Subject", "") or "Kein Betreff"
|
||||
date_str = msg.get("Date", "")
|
||||
message_id = msg.get("Message-ID", "")
|
||||
in_reply_to = msg.get("In-Reply-To", "")
|
||||
|
||||
recipients = _format_recipients(to_raw)
|
||||
cc_recipients = _format_recipients(cc_raw)
|
||||
email_date = _parse_email_date(date_str)
|
||||
|
||||
body = _extract_body(msg)
|
||||
attachments = _extract_attachments(msg)
|
||||
|
||||
# Build artifact
|
||||
return self._build_artifact(
|
||||
file_path=file_path,
|
||||
sender=sender,
|
||||
recipients=recipients,
|
||||
cc_recipients=cc_recipients,
|
||||
subject=subject,
|
||||
email_date=email_date,
|
||||
body=body,
|
||||
attachments=attachments,
|
||||
message_id=message_id,
|
||||
in_reply_to=in_reply_to,
|
||||
source_name=source_name,
|
||||
)
|
||||
|
||||
def _process_msg(self, file_path: Path, source_name: str) -> Artifact:
|
||||
"""Processes a .msg file. Requires extract_msg package."""
|
||||
try:
|
||||
import extract_msg # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
f"Das Paket 'extract_msg' wird für .msg-Dateien benötigt. "
|
||||
f"Installation: pip install extract-msg"
|
||||
)
|
||||
|
||||
msg = extract_msg.Message(str(file_path))
|
||||
try:
|
||||
sender = msg.sender or ""
|
||||
recipients = [r.strip() for r in (msg.to or "").split(";") if r.strip()]
|
||||
cc_recipients = [r.strip() for r in (msg.cc or "").split(";") if r.strip()]
|
||||
subject = msg.subject or "Kein Betreff"
|
||||
body = msg.body or ""
|
||||
email_date = msg.date.date() if msg.date else None
|
||||
message_id = msg.messageId or ""
|
||||
|
||||
attachments = [
|
||||
{"filename": att.longFilename or att.shortFilename or "unnamed",
|
||||
"content_type": att.mimetype or "application/octet-stream"}
|
||||
for att in (msg.attachments or [])
|
||||
]
|
||||
finally:
|
||||
msg.close()
|
||||
|
||||
return self._build_artifact(
|
||||
file_path=file_path,
|
||||
sender=sender,
|
||||
recipients=recipients,
|
||||
cc_recipients=cc_recipients,
|
||||
subject=subject,
|
||||
email_date=email_date,
|
||||
body=body,
|
||||
attachments=attachments,
|
||||
message_id=message_id,
|
||||
in_reply_to="",
|
||||
source_name=source_name,
|
||||
)
|
||||
|
||||
def _build_artifact(
|
||||
self,
|
||||
file_path: Path,
|
||||
sender: str,
|
||||
recipients: list[str],
|
||||
cc_recipients: list[str],
|
||||
subject: str,
|
||||
email_date: date | None,
|
||||
body: str,
|
||||
attachments: list[dict[str, str]],
|
||||
message_id: str,
|
||||
in_reply_to: str,
|
||||
source_name: str,
|
||||
) -> Artifact:
|
||||
"""Builds a Markdown artifact from parsed email data."""
|
||||
# Build Markdown content
|
||||
content_lines: list[str] = []
|
||||
content_lines.append(f"# {subject}\n")
|
||||
content_lines.append("")
|
||||
content_lines.append(f"**Von:** {sender} ")
|
||||
content_lines.append(f"**An:** {', '.join(recipients)} ")
|
||||
if cc_recipients:
|
||||
content_lines.append(f"**CC:** {', '.join(cc_recipients)} ")
|
||||
if email_date:
|
||||
content_lines.append(f"**Datum:** {email_date.isoformat()} ")
|
||||
content_lines.append("")
|
||||
content_lines.append("---\n")
|
||||
content_lines.append("")
|
||||
content_lines.append(body.strip())
|
||||
|
||||
# Attachments section
|
||||
if attachments:
|
||||
content_lines.append("")
|
||||
content_lines.append("")
|
||||
content_lines.append("## Anhänge\n")
|
||||
content_lines.append("")
|
||||
for att in attachments:
|
||||
filename = att["filename"]
|
||||
content_lines.append(f"- [{filename}](attachments/{filename})")
|
||||
|
||||
content = "\n".join(content_lines)
|
||||
|
||||
# Build source dict for frontmatter
|
||||
source_dict: dict[str, str] = {
|
||||
"type": "email",
|
||||
"file": str(file_path),
|
||||
}
|
||||
if message_id:
|
||||
source_dict["message_id"] = message_id
|
||||
|
||||
# Build metadata
|
||||
metadata = ArtifactMetadata(
|
||||
type="email",
|
||||
title=subject,
|
||||
tags=["email"],
|
||||
source_context="",
|
||||
created=email_date,
|
||||
updated=email_date,
|
||||
shareable=False,
|
||||
source=source_dict,
|
||||
category="inbox",
|
||||
people=[sender] + recipients,
|
||||
content_hash=_compute_content_hash(content),
|
||||
)
|
||||
|
||||
# Generate a filename from subject
|
||||
date_prefix = email_date.isoformat() if email_date else "undated"
|
||||
slug = _slugify(subject)
|
||||
filename = f"{date_prefix}-{slug}.md"
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=Path(filename),
|
||||
)
|
||||
@@ -0,0 +1,483 @@
|
||||
"""Jira-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Extrahiert Issues via JQL-Filter aus der Jira REST API und liefert
|
||||
sie als strukturierte Markdown-Artefakte zurück.
|
||||
|
||||
Unterstützt inkrementelle Updates via das `updated`-Feld der Jira-Issues.
|
||||
|
||||
Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["JiraSource"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hilfsfunktionen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_content_hash(text: str) -> str:
|
||||
"""Berechnet SHA-256-Hash eines Textes."""
|
||||
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
def _parse_jira_date(date_str: str | None) -> date | None:
|
||||
"""Parsed ein Jira-Datumsfeld (ISO-8601) in ein date-Objekt.
|
||||
|
||||
Jira liefert Timestamps wie '2025-07-01T10:30:00.000+0200'.
|
||||
"""
|
||||
if not date_str:
|
||||
return None
|
||||
try:
|
||||
# Versuche volles Datetime-Format
|
||||
dt = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
|
||||
return dt.date()
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
try:
|
||||
return date.fromisoformat(date_str[:10])
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Erzeugt einen URL-freundlichen Slug aus einem Text."""
|
||||
import re
|
||||
|
||||
slug = text.lower().strip()
|
||||
slug = re.sub(r"[^a-z0-9\-]+", "-", slug)
|
||||
slug = re.sub(r"-+", "-", slug)
|
||||
return slug.strip("-")
|
||||
|
||||
|
||||
def _build_issue_markdown(issue: dict[str, Any]) -> str:
|
||||
"""Stellt ein Jira-Issue als Markdown-Body zusammen.
|
||||
|
||||
Enthält: Summary als H1, Description, Kommentare als Sub-Sections.
|
||||
"""
|
||||
fields = issue.get("fields", {})
|
||||
summary = fields.get("summary", "Untitled Issue")
|
||||
description = fields.get("description") or ""
|
||||
status = fields.get("status", {}).get("name", "Unknown")
|
||||
|
||||
parts: list[str] = []
|
||||
|
||||
# Title
|
||||
parts.append(f"# {summary}\n")
|
||||
|
||||
# Status-Info
|
||||
parts.append(f"**Status:** {status}\n")
|
||||
|
||||
# Description
|
||||
if description:
|
||||
parts.append("## Description\n")
|
||||
parts.append(f"{description}\n")
|
||||
|
||||
# Kommentare
|
||||
comment_data = fields.get("comment", {})
|
||||
comments = comment_data.get("comments", []) if isinstance(comment_data, dict) else []
|
||||
|
||||
if comments:
|
||||
parts.append("## Comments\n")
|
||||
for comment in comments:
|
||||
author = comment.get("author", {}).get("displayName", "Unknown")
|
||||
created = comment.get("created", "")[:10]
|
||||
body = comment.get("body", "")
|
||||
parts.append(f"### {author} ({created})\n")
|
||||
parts.append(f"{body}\n")
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _build_frontmatter(issue: dict[str, Any], base_url: str) -> dict[str, str]:
|
||||
"""Erstellt die Frontmatter-Source-Felder für ein Jira-Issue."""
|
||||
fields = issue.get("fields", {})
|
||||
key = issue.get("key", "")
|
||||
project = fields.get("project", {}).get("key", "")
|
||||
status = fields.get("status", {}).get("name", "")
|
||||
assignee = fields.get("assignee", {})
|
||||
reporter = fields.get("reporter", {})
|
||||
|
||||
assignee_name = assignee.get("displayName", "") if assignee else ""
|
||||
reporter_name = reporter.get("displayName", "") if reporter else ""
|
||||
|
||||
# Construct browse URL
|
||||
jira_url = f"{base_url.rstrip('/')}/browse/{key}" if base_url else ""
|
||||
|
||||
return {
|
||||
"type": "jira",
|
||||
"issue_key": key,
|
||||
"project": project,
|
||||
"status": status,
|
||||
"assignee": assignee_name,
|
||||
"reporter": reporter_name,
|
||||
"url": jira_url,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JiraSource – implementiert SourceStrategy ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class JiraSource(SourceStrategy):
|
||||
"""Quellstrategie: Jira-Issues via JQL-Filter extrahieren.
|
||||
|
||||
Verwendet die Jira REST API (v2) um Issues basierend auf einem
|
||||
konfigurierbaren JQL-Filter abzurufen und als Markdown-Artefakte
|
||||
zurückzugeben.
|
||||
|
||||
Unterstützt inkrementelle Verarbeitung: Beim wiederholten Aufruf
|
||||
können über `_last_updated` bereits bekannte Update-Zeitstempel
|
||||
übergeben werden, um nur geänderte Issues zu verarbeiten.
|
||||
|
||||
SourceConfig params:
|
||||
base_url: Jira-Instanz-URL (z.B. "https://jira.bahn.de")
|
||||
api_key / token: API-Token für Authentifizierung
|
||||
username: Benutzername für Basic Auth (optional)
|
||||
jql: JQL-Filter-String
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialisiert die JiraSource."""
|
||||
self._last_updated: dict[str, str] = {}
|
||||
self._session: requests.Session | None = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SourceStrategy ABC Interface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
|
||||
"""Extrahiert Jira-Issues basierend auf der Quellkonfiguration.
|
||||
|
||||
Args:
|
||||
config: SourceConfig mit Jira-Verbindungsparametern.
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
params = config.params
|
||||
base_url = params.get("base_url", "")
|
||||
token = params.get("api_key") or params.get("token", "")
|
||||
username = params.get("username", "")
|
||||
jql = params.get("jql", "")
|
||||
|
||||
if not base_url:
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="config",
|
||||
message="Missing required param 'base_url'",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
if not jql:
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="config",
|
||||
message="Missing required param 'jql'",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
# Session aufbauen
|
||||
session = self._create_session(token, username)
|
||||
|
||||
# Issues abrufen
|
||||
try:
|
||||
issues = self._fetch_issues(session, base_url, jql)
|
||||
except JiraAPIError as e:
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type=e.error_type,
|
||||
message=str(e),
|
||||
timestamp="",
|
||||
retry=e.retry,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
# Issues zu Artefakten konvertieren
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
skipped = 0
|
||||
|
||||
for issue in issues:
|
||||
key = issue.get("key", "unknown")
|
||||
fields = issue.get("fields", {})
|
||||
updated_str = fields.get("updated", "")
|
||||
|
||||
# Inkrementelle Verarbeitung: Skip wenn nicht geändert
|
||||
if self._last_updated.get(key) == updated_str and updated_str:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
artifact = self._issue_to_artifact(issue, base_url, context)
|
||||
artifacts.append(artifact)
|
||||
# Update-Timestamp merken
|
||||
if updated_str:
|
||||
self._last_updated[key] = updated_str
|
||||
except Exception as e:
|
||||
logger.warning("Fehler beim Verarbeiten von %s: %s", key, e)
|
||||
errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Error processing issue {key}: {e}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
artifact_path=key,
|
||||
)
|
||||
)
|
||||
|
||||
return ExtractionResult(
|
||||
artifacts=artifacts,
|
||||
errors=errors,
|
||||
skipped=skipped,
|
||||
)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Jira unterstützt inkrementelle Verarbeitung via updated-Feld.
|
||||
|
||||
Returns:
|
||||
True – inkrementelle Verarbeitung wird unterstützt.
|
||||
"""
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inkrementelle Verarbeitung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def set_last_updated(self, timestamps: dict[str, str]) -> None:
|
||||
"""Setzt die bekannten Update-Timestamps für inkrementelle Verarbeitung.
|
||||
|
||||
Args:
|
||||
timestamps: Mapping von Issue-Key zu letztem `updated`-Timestamp.
|
||||
"""
|
||||
self._last_updated = dict(timestamps)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Implementierung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _create_session(self, token: str, username: str) -> requests.Session:
|
||||
"""Erstellt eine konfigurierte HTTP-Session für Jira.
|
||||
|
||||
Args:
|
||||
token: API-Token oder Passwort.
|
||||
username: Benutzername (für Basic Auth).
|
||||
|
||||
Returns:
|
||||
Konfigurierte requests.Session.
|
||||
"""
|
||||
session = requests.Session()
|
||||
session.headers["Accept"] = "application/json"
|
||||
session.headers["Content-Type"] = "application/json"
|
||||
|
||||
if username and token:
|
||||
# Basic Auth (Jira Server/DC)
|
||||
session.auth = (username, token)
|
||||
elif token:
|
||||
# Bearer Token (Jira Cloud)
|
||||
session.headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
return session
|
||||
|
||||
def _fetch_issues(
|
||||
self,
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
jql: str,
|
||||
max_results: int = 100,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Ruft Issues aus der Jira REST API ab.
|
||||
|
||||
Paginiert automatisch bei großen Ergebnismengen.
|
||||
|
||||
Args:
|
||||
session: Die HTTP-Session.
|
||||
base_url: Jira-Instanz-URL.
|
||||
jql: JQL-Filter.
|
||||
max_results: Maximale Anzahl pro Request.
|
||||
|
||||
Returns:
|
||||
Liste von Issue-Dictionaries.
|
||||
|
||||
Raises:
|
||||
JiraAPIError: Bei API-Fehlern.
|
||||
"""
|
||||
url = f"{base_url.rstrip('/')}/rest/api/2/search"
|
||||
all_issues: list[dict[str, Any]] = []
|
||||
start_at = 0
|
||||
|
||||
while True:
|
||||
params = {
|
||||
"jql": jql,
|
||||
"startAt": start_at,
|
||||
"maxResults": max_results,
|
||||
"fields": "summary,description,status,assignee,reporter,"
|
||||
"project,comment,updated,created",
|
||||
}
|
||||
|
||||
try:
|
||||
response = session.get(url, params=params, timeout=30)
|
||||
except requests.ConnectionError as e:
|
||||
raise JiraAPIError(
|
||||
f"Connection failed to {base_url}: {e}",
|
||||
error_type="connection",
|
||||
retry=True,
|
||||
)
|
||||
except requests.Timeout as e:
|
||||
raise JiraAPIError(
|
||||
f"Request timeout to {base_url}: {e}",
|
||||
error_type="connection",
|
||||
retry=True,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
raise JiraAPIError(
|
||||
f"Request error to {base_url}: {e}",
|
||||
error_type="connection",
|
||||
retry=True,
|
||||
)
|
||||
|
||||
if response.status_code == 401:
|
||||
raise JiraAPIError(
|
||||
f"Authentication failed for {base_url}",
|
||||
error_type="auth",
|
||||
retry=False,
|
||||
)
|
||||
elif response.status_code == 403:
|
||||
raise JiraAPIError(
|
||||
f"Access forbidden for {base_url}",
|
||||
error_type="auth",
|
||||
retry=False,
|
||||
)
|
||||
elif response.status_code >= 400:
|
||||
raise JiraAPIError(
|
||||
f"API error {response.status_code} from {base_url}: "
|
||||
f"{response.text[:200]}",
|
||||
error_type="connection",
|
||||
retry=response.status_code >= 500,
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
issues = data.get("issues", [])
|
||||
all_issues.extend(issues)
|
||||
|
||||
# Pagination
|
||||
total = data.get("total", 0)
|
||||
start_at += len(issues)
|
||||
if start_at >= total or not issues:
|
||||
break
|
||||
|
||||
return all_issues
|
||||
|
||||
def _issue_to_artifact(
|
||||
self, issue: dict[str, Any], base_url: str, context: str
|
||||
) -> Artifact:
|
||||
"""Konvertiert ein Jira-Issue-Dict in ein Artifact.
|
||||
|
||||
Args:
|
||||
issue: Das Issue-Dictionary von der Jira API.
|
||||
base_url: Jira-Instanz-URL für Link-Generierung.
|
||||
context: Der Zielkontext.
|
||||
|
||||
Returns:
|
||||
Ein Artifact mit Markdown-Inhalt und angereichertem Frontmatter.
|
||||
"""
|
||||
fields = issue.get("fields", {})
|
||||
key = issue.get("key", "unknown")
|
||||
summary = fields.get("summary", "Untitled")
|
||||
|
||||
# Markdown-Body zusammenstellen
|
||||
content = _build_issue_markdown(issue)
|
||||
|
||||
# Content-Hash
|
||||
content_hash = _compute_content_hash(content)
|
||||
|
||||
# Source-Frontmatter
|
||||
source_dict = _build_frontmatter(issue, base_url)
|
||||
|
||||
# Daten parsen
|
||||
created = _parse_jira_date(fields.get("created"))
|
||||
updated = _parse_jira_date(fields.get("updated"))
|
||||
|
||||
# Tags aus Projekt und Status ableiten
|
||||
project_key = fields.get("project", {}).get("key", "")
|
||||
status_name = fields.get("status", {}).get("name", "")
|
||||
tags: list[str] = []
|
||||
if project_key:
|
||||
tags.append(project_key.lower())
|
||||
if status_name:
|
||||
tags.append(_slugify(status_name))
|
||||
|
||||
# Metadata zusammenbauen
|
||||
metadata = ArtifactMetadata(
|
||||
type="jira",
|
||||
title=summary,
|
||||
tags=tags,
|
||||
source_context=context,
|
||||
created=created,
|
||||
updated=updated,
|
||||
shareable=False,
|
||||
content_hash=content_hash,
|
||||
source=source_dict,
|
||||
category="project",
|
||||
)
|
||||
|
||||
# Dateiname aus Issue-Key ableiten
|
||||
from pathlib import Path
|
||||
|
||||
filename = f"{key.lower()}.md"
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=Path(filename),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fehlerklasse
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class JiraAPIError(Exception):
|
||||
"""Fehler bei der Kommunikation mit der Jira REST API."""
|
||||
|
||||
def __init__(self, message: str, error_type: str = "connection", retry: bool = True):
|
||||
super().__init__(message)
|
||||
self.error_type = error_type
|
||||
self.retry = retry
|
||||
@@ -0,0 +1,426 @@
|
||||
"""Link-Source und LinkRegistry für Web-Link-Management.
|
||||
|
||||
Verwaltet Web-Links als Wissensartefakte vom Typ `link` im `links/`-Unterordner
|
||||
des jeweiligen Kontext-Knowledge-Folders. Unterstützt URL-Deduplication mit
|
||||
Tag-Merging, Metadaten-Fetch und Suche über URL, Titel und Tags.
|
||||
|
||||
Requirements: 8.1, 8.2, 8.4, 8.5, 8.6, 8.7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
Artifact,
|
||||
ArtifactMetadata,
|
||||
parse_frontmatter,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LinkRegistry:
|
||||
"""Manages web links as knowledge artifacts.
|
||||
|
||||
Links are stored as Markdown files with YAML-Frontmatter in
|
||||
`{context_path}/links/`. Each link artifact has type: link and contains
|
||||
url, title, tags, created in its frontmatter.
|
||||
|
||||
Supports:
|
||||
- URL deduplication with tag merging (Req 8.7)
|
||||
- Metadata fetching from target URL (Req 8.4)
|
||||
- Search over URL, title, and tags (Req 8.6)
|
||||
- fetch_failed flag when URL is unreachable (Req 8.5)
|
||||
"""
|
||||
|
||||
def __init__(self, context_path: Path) -> None:
|
||||
"""Initialize LinkRegistry with a context knowledge path.
|
||||
|
||||
Args:
|
||||
context_path: Path to the context's knowledge folder,
|
||||
e.g. `{monorepo_root}/bahn/knowledge/`.
|
||||
"""
|
||||
self.context_path = context_path
|
||||
self.links_dir = context_path / "links"
|
||||
|
||||
def save_link(
|
||||
self,
|
||||
url: str,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
) -> Path:
|
||||
"""Save or update a link artifact. Deduplicates by URL.
|
||||
|
||||
If a link with the same URL already exists, merges tags (union of
|
||||
existing + new tags) instead of creating a duplicate.
|
||||
|
||||
Args:
|
||||
url: The web URL to save.
|
||||
title: Optional title override. If None, fetched from URL.
|
||||
tags: Optional list of tags.
|
||||
|
||||
Returns:
|
||||
Path of the created or updated link artifact.
|
||||
"""
|
||||
if tags is None:
|
||||
tags = []
|
||||
|
||||
# Check for existing link with same URL
|
||||
existing_path = self._find_existing(url)
|
||||
|
||||
if existing_path is not None:
|
||||
return self._merge_tags(existing_path, tags)
|
||||
|
||||
# Fetch metadata if title not provided
|
||||
fetched_title = ""
|
||||
fetched_description = ""
|
||||
fetch_failed = False
|
||||
|
||||
if title is None:
|
||||
fetched_title, fetched_description = self._fetch_metadata(url)
|
||||
if fetched_title:
|
||||
title = fetched_title
|
||||
else:
|
||||
# Use domain + path as fallback title
|
||||
title = self._title_from_url(url)
|
||||
fetch_failed = True
|
||||
else:
|
||||
# Still try to fetch for description, but don't override title
|
||||
try:
|
||||
_, fetched_description = self._fetch_metadata(url)
|
||||
except Exception:
|
||||
fetch_failed = True
|
||||
|
||||
# Generate filename from title
|
||||
slug = self._slugify(title)
|
||||
if not slug:
|
||||
slug = self._slugify(self._title_from_url(url))
|
||||
if not slug:
|
||||
slug = "untitled-link"
|
||||
filename = f"{slug}.md"
|
||||
|
||||
# Ensure links directory exists
|
||||
self.links_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Avoid filename collisions
|
||||
target_path = self.links_dir / filename
|
||||
counter = 1
|
||||
while target_path.exists():
|
||||
target_path = self.links_dir / f"{slug}-{counter}.md"
|
||||
counter += 1
|
||||
|
||||
# Build the artifact content
|
||||
content = self._build_link_content(
|
||||
url=url,
|
||||
title=title,
|
||||
tags=tags,
|
||||
description=fetched_description,
|
||||
fetch_failed=fetch_failed,
|
||||
)
|
||||
|
||||
target_path.write_text(content, encoding="utf-8")
|
||||
return target_path
|
||||
|
||||
def search_links(self, query: str) -> list[Artifact]:
|
||||
"""Search links by URL, title, or tags.
|
||||
|
||||
Performs case-insensitive substring matching against the URL,
|
||||
title, and all tags of each link artifact.
|
||||
|
||||
Args:
|
||||
query: Search string to match against link metadata.
|
||||
|
||||
Returns:
|
||||
List of matching Artifact objects.
|
||||
"""
|
||||
if not self.links_dir.exists():
|
||||
return []
|
||||
|
||||
query_lower = query.lower()
|
||||
results: list[Artifact] = []
|
||||
|
||||
for md_file in sorted(self.links_dir.glob("*.md")):
|
||||
try:
|
||||
artifact = parse_frontmatter(md_file)
|
||||
except (ValueError, OSError) as e:
|
||||
logger.warning("Skipping unparseable link file %s: %s", md_file, e)
|
||||
continue
|
||||
|
||||
# Only consider link-type artifacts
|
||||
if artifact.metadata.type != "link":
|
||||
continue
|
||||
|
||||
# Match against URL
|
||||
url = artifact.metadata.source.get("url", "")
|
||||
if query_lower in url.lower():
|
||||
results.append(artifact)
|
||||
continue
|
||||
|
||||
# Match against title
|
||||
if query_lower in artifact.metadata.title.lower():
|
||||
results.append(artifact)
|
||||
continue
|
||||
|
||||
# Match against tags
|
||||
if any(query_lower in tag.lower() for tag in artifact.metadata.tags):
|
||||
results.append(artifact)
|
||||
continue
|
||||
|
||||
return results
|
||||
|
||||
def _fetch_metadata(self, url: str) -> tuple[str, str]:
|
||||
"""Fetch page title and description from URL.
|
||||
|
||||
Makes an HTTP GET request to the URL and tries to extract
|
||||
the <title> tag and meta description.
|
||||
|
||||
Args:
|
||||
url: The target URL to fetch.
|
||||
|
||||
Returns:
|
||||
Tuple of (title, description). Empty strings if fetch fails.
|
||||
"""
|
||||
try:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": "MonorepoCLI/1.0 LinkRegistry"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
# Read limited content (first 64KB) to avoid huge downloads
|
||||
raw = response.read(65536)
|
||||
charset = response.headers.get_content_charset() or "utf-8"
|
||||
try:
|
||||
html = raw.decode(charset, errors="replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
html = raw.decode("utf-8", errors="replace")
|
||||
|
||||
title = self._extract_html_title(html)
|
||||
description = self._extract_meta_description(html)
|
||||
return title, description
|
||||
|
||||
except Exception as e:
|
||||
logger.debug("Failed to fetch metadata from %s: %s", url, e)
|
||||
return "", ""
|
||||
|
||||
def _find_existing(self, url: str) -> Path | None:
|
||||
"""Check for duplicate URL in context links folder.
|
||||
|
||||
Scans all Markdown files in the links directory and checks
|
||||
if any has a matching URL in its frontmatter source.url field.
|
||||
|
||||
Args:
|
||||
url: The URL to check for duplicates.
|
||||
|
||||
Returns:
|
||||
Path to the existing artifact, or None if not found.
|
||||
"""
|
||||
if not self.links_dir.exists():
|
||||
return None
|
||||
|
||||
for md_file in self.links_dir.glob("*.md"):
|
||||
try:
|
||||
artifact = parse_frontmatter(md_file)
|
||||
except (ValueError, OSError):
|
||||
continue
|
||||
|
||||
if artifact.metadata.type != "link":
|
||||
continue
|
||||
|
||||
existing_url = artifact.metadata.source.get("url", "")
|
||||
if existing_url == url:
|
||||
return md_file
|
||||
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _merge_tags(self, existing_path: Path, new_tags: list[str]) -> Path:
|
||||
"""Merge new tags into an existing link artifact.
|
||||
|
||||
Reads the existing artifact, computes union of tags,
|
||||
and rewrites the file if tags changed.
|
||||
|
||||
Args:
|
||||
existing_path: Path to the existing link artifact.
|
||||
new_tags: New tags to merge in.
|
||||
|
||||
Returns:
|
||||
Path to the updated artifact.
|
||||
"""
|
||||
artifact = parse_frontmatter(existing_path)
|
||||
|
||||
# Compute tag union (preserving order, existing first)
|
||||
existing_tags = list(artifact.metadata.tags)
|
||||
merged = list(existing_tags)
|
||||
for tag in new_tags:
|
||||
if tag not in merged:
|
||||
merged.append(tag)
|
||||
|
||||
if merged == existing_tags:
|
||||
# No change needed
|
||||
return existing_path
|
||||
|
||||
# Rewrite with updated tags
|
||||
artifact.metadata.tags = merged
|
||||
content = self._render_artifact(artifact)
|
||||
existing_path.write_text(content, encoding="utf-8")
|
||||
return existing_path
|
||||
|
||||
def _render_artifact(self, artifact: Artifact) -> str:
|
||||
"""Render an artifact back to its file content (frontmatter + body).
|
||||
|
||||
Args:
|
||||
artifact: The artifact to render.
|
||||
|
||||
Returns:
|
||||
Complete file content as string.
|
||||
"""
|
||||
from monorepo.knowledge.artifact import _generate_frontmatter
|
||||
|
||||
frontmatter = _generate_frontmatter(artifact.metadata)
|
||||
return frontmatter + artifact.content
|
||||
|
||||
def _build_link_content(
|
||||
self,
|
||||
url: str,
|
||||
title: str,
|
||||
tags: list[str],
|
||||
description: str,
|
||||
fetch_failed: bool,
|
||||
) -> str:
|
||||
"""Build the complete Markdown content for a link artifact.
|
||||
|
||||
Args:
|
||||
url: The link URL.
|
||||
title: The link title.
|
||||
tags: List of tags.
|
||||
description: Fetched description (may be empty).
|
||||
fetch_failed: Whether metadata fetch failed.
|
||||
|
||||
Returns:
|
||||
Complete Markdown file content with frontmatter.
|
||||
"""
|
||||
today = date.today().isoformat()
|
||||
|
||||
# Build frontmatter data
|
||||
data: dict = {
|
||||
"type": "link",
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
"created": today,
|
||||
"source": {"type": "link", "url": url},
|
||||
"source_context": "",
|
||||
"shareable": False,
|
||||
}
|
||||
|
||||
if fetch_failed:
|
||||
data["fetch_failed"] = True
|
||||
|
||||
# Serialize frontmatter
|
||||
yaml_str = yaml.dump(
|
||||
data,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
frontmatter = f"---\n{yaml_str}---\n"
|
||||
|
||||
# Build body
|
||||
body_lines = [f"\n[{title}]({url})\n"]
|
||||
if description:
|
||||
body_lines.append(f"\n{description}\n")
|
||||
|
||||
body = "".join(body_lines)
|
||||
return frontmatter + body
|
||||
|
||||
def _title_from_url(self, url: str) -> str:
|
||||
"""Derive a title from a URL (domain + path).
|
||||
|
||||
Args:
|
||||
url: The URL to derive title from.
|
||||
|
||||
Returns:
|
||||
A human-readable title string.
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
path_part = parsed.path.strip("/")
|
||||
if path_part:
|
||||
return f"{parsed.netloc}/{path_part}"
|
||||
return parsed.netloc or url
|
||||
|
||||
def _slugify(self, text: str) -> str:
|
||||
"""Convert text to URL-safe slug.
|
||||
|
||||
Lowercase, replace non-alphanumeric with hyphens,
|
||||
collapse multiple hyphens, limit to 60 chars.
|
||||
|
||||
Args:
|
||||
text: The text to slugify.
|
||||
|
||||
Returns:
|
||||
A URL-safe slug string.
|
||||
"""
|
||||
slug = text.lower()
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
||||
slug = re.sub(r"-{2,}", "-", slug)
|
||||
slug = slug.strip("-")
|
||||
return slug[:60]
|
||||
|
||||
def _extract_html_title(self, html: str) -> str:
|
||||
"""Extract <title> content from HTML.
|
||||
|
||||
Args:
|
||||
html: The HTML content string.
|
||||
|
||||
Returns:
|
||||
The title text, or empty string if not found.
|
||||
"""
|
||||
match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# Clean up HTML entities
|
||||
title = re.sub(r"&", "&", title)
|
||||
title = re.sub(r"<", "<", title)
|
||||
title = re.sub(r">", ">", title)
|
||||
title = re.sub(r""", '"', title)
|
||||
title = re.sub(r"'", "'", title)
|
||||
title = re.sub(r"\s+", " ", title)
|
||||
return title
|
||||
return ""
|
||||
|
||||
def _extract_meta_description(self, html: str) -> str:
|
||||
"""Extract meta description from HTML.
|
||||
|
||||
Args:
|
||||
html: The HTML content string.
|
||||
|
||||
Returns:
|
||||
The description text, or empty string if not found.
|
||||
"""
|
||||
match = re.search(
|
||||
r'<meta\s+[^>]*name=["\']description["\'][^>]*content=["\']([^"\']*)["\']',
|
||||
html,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if not match:
|
||||
# Try alternative ordering (content before name)
|
||||
match = re.search(
|
||||
r'<meta\s+[^>]*content=["\']([^"\']*)["\'][^>]*name=["\']description["\']',
|
||||
html,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ""
|
||||
@@ -1,111 +1,376 @@
|
||||
"""Markdown-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Scannt ein Verzeichnis rekursiv nach .md-Dateien mit YAML-Frontmatter
|
||||
Scannt ein Verzeichnis rekursiv nach .md/.txt-Dateien mit YAML-Frontmatter
|
||||
und liefert eine Liste von Artifacts zurück.
|
||||
|
||||
Requirements: 3.2, 3.10, 3.12, 3.19
|
||||
Implementiert das SourceStrategy-Interface (ABC) und unterstützt
|
||||
inkrementelle Verarbeitung via Content-Hash.
|
||||
|
||||
Requirements: 2.2, 2.4, 2.7, 4.1
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, parse_frontmatter
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata, parse_frontmatter
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError as BaseSourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legacy-kompatibles SourceError (behält file_path/error-Interface)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceError:
|
||||
"""Fehler bei der Verarbeitung einer einzelnen Quelldatei."""
|
||||
"""Fehler bei der Verarbeitung einer einzelnen Quelldatei.
|
||||
|
||||
Behält das alte Interface (file_path, error, retry) für Abwärtskompatibilität
|
||||
mit ETLPipeline und bestehenden Tests.
|
||||
"""
|
||||
|
||||
file_path: str
|
||||
error: str
|
||||
retry: bool = True
|
||||
|
||||
def to_base_error(self, source_name: str = "markdown") -> BaseSourceError:
|
||||
"""Konvertiert in das neue BaseSourceError-Format."""
|
||||
return BaseSourceError(
|
||||
source_name=source_name,
|
||||
error_type="parse",
|
||||
message=self.error,
|
||||
timestamp="",
|
||||
retry=self.retry,
|
||||
artifact_path=self.file_path,
|
||||
)
|
||||
|
||||
@dataclass
|
||||
class MarkdownSource:
|
||||
"""Quellstrategie: Rekursives Scannen eines Verzeichnisses nach Markdown-Dateien.
|
||||
|
||||
Findet alle .md-Dateien im angegebenen Verzeichnis (rekursiv),
|
||||
__all__ = ["MarkdownSource", "SourceError"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content-Hash-Hilfsfunktion für inkrementelle Verarbeitung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_file_hash(file_path: Path) -> str:
|
||||
"""Berechnet einen SHA-256-Hash des Dateiinhalts.
|
||||
|
||||
Wird für inkrementelle Verarbeitung verwendet – nur geänderte
|
||||
Dateien werden erneut verarbeitet.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
|
||||
Returns:
|
||||
Hash-String im Format "sha256:<hex>".
|
||||
"""
|
||||
content = file_path.read_bytes()
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MarkdownSource – implementiert SourceStrategy ABC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Unterstützte Dateiendungen
|
||||
_SUPPORTED_EXTENSIONS = {".md", ".txt"}
|
||||
|
||||
|
||||
class MarkdownSource(SourceStrategy):
|
||||
"""Quellstrategie: Rekursives Scannen eines Verzeichnisses nach Markdown/Text-Dateien.
|
||||
|
||||
Findet alle .md- und .txt-Dateien im angegebenen Verzeichnis (rekursiv),
|
||||
parsed jede Datei mit parse_frontmatter und liefert die resultierenden
|
||||
Artifacts. Dateien ohne gültiges YAML-Frontmatter werden übersprungen
|
||||
und als Warning geloggt.
|
||||
Artifacts. Dateien ohne gültiges YAML-Frontmatter werden als Plain-Text-
|
||||
Artefakte mit minimalem Frontmatter behandelt (.txt) oder als Fehler
|
||||
geloggt (.md).
|
||||
|
||||
Implementiert das SourceStrategy-Interface und unterstützt zusätzlich
|
||||
die alte Convenience-Schnittstelle `MarkdownSource(directory=...)`.
|
||||
|
||||
Inkrementelle Verarbeitung:
|
||||
Bei wiederholtem Aufruf können über `_known_hashes` bereits bekannte
|
||||
Datei-Hashes übergeben werden. Dateien mit unverändertem Hash werden
|
||||
übersprungen (gezählt in `ExtractionResult.skipped`).
|
||||
"""
|
||||
|
||||
directory: Path
|
||||
errors: list[SourceError] = field(default_factory=list)
|
||||
def __init__(self, directory: Path | None = None) -> None:
|
||||
"""Initialisiert die MarkdownSource.
|
||||
|
||||
def extract(self) -> list[Artifact]:
|
||||
"""Extrahiert alle Artefakte aus dem Quellverzeichnis.
|
||||
Args:
|
||||
directory: Optionaler Pfad zum Quellverzeichnis (Convenience-Interface).
|
||||
Wenn angegeben, kann extract() ohne config aufgerufen werden.
|
||||
"""
|
||||
self._directory: Path | None = directory
|
||||
self.errors: list[SourceError] = []
|
||||
self._known_hashes: dict[str, str] = {}
|
||||
|
||||
Durchsucht das Verzeichnis rekursiv nach .md-Dateien,
|
||||
parsed jede Datei und sammelt Fehler ohne abzubrechen.
|
||||
# ------------------------------------------------------------------
|
||||
# SourceStrategy ABC Interface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def extract( # type: ignore[override]
|
||||
self,
|
||||
config: SourceConfig | None = None,
|
||||
context: str = "",
|
||||
) -> ExtractionResult | list[Artifact]:
|
||||
"""Extrahiert Artefakte aus dem konfigurierten Verzeichnis.
|
||||
|
||||
Unterstützt zwei Aufrufmodi:
|
||||
1. Neues Interface: extract(config, context) → ExtractionResult
|
||||
2. Legacy-Interface: extract() → list[Artifact]
|
||||
|
||||
Args:
|
||||
config: Die Quellkonfiguration mit Verzeichnis in params["directory"].
|
||||
Wenn None, wird self._directory verwendet (Legacy-Modus).
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
Liste erfolgreich geparster Artifacts.
|
||||
ExtractionResult (neues Interface) oder list[Artifact] (Legacy).
|
||||
"""
|
||||
self.errors = []
|
||||
artifacts: list[Artifact] = []
|
||||
# Bestimme ob neues oder altes Interface verwendet wird
|
||||
use_legacy = config is None
|
||||
|
||||
if not self.directory.exists():
|
||||
logger.warning(
|
||||
"Quellverzeichnis existiert nicht: %s", self.directory
|
||||
# Verzeichnis auflösen
|
||||
directory = self._resolve_directory(config)
|
||||
|
||||
# Fehler-Liste zurücksetzen
|
||||
self.errors = []
|
||||
|
||||
# Extraktion durchführen
|
||||
artifacts, errors, skipped = self._do_extract(directory)
|
||||
|
||||
# Fehler speichern (für Legacy-Kompatibilität)
|
||||
self.errors = errors
|
||||
|
||||
if use_legacy:
|
||||
# Legacy-Rückgabe: nur die Artefaktliste
|
||||
return artifacts
|
||||
else:
|
||||
# Neues Interface: ExtractionResult (konvertiere lokale Fehler)
|
||||
base_errors = [e.to_base_error() for e in errors]
|
||||
return ExtractionResult(
|
||||
artifacts=artifacts,
|
||||
errors=base_errors,
|
||||
skipped=skipped,
|
||||
)
|
||||
self.errors.append(
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Markdown/Text-Quellen unterstützen inkrementelle Verarbeitung.
|
||||
|
||||
Über Content-Hashes werden nur geänderte Dateien erneut verarbeitet.
|
||||
|
||||
Returns:
|
||||
True – inkrementelle Verarbeitung wird unterstützt.
|
||||
"""
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inkrementelle Verarbeitung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def set_known_hashes(self, hashes: dict[str, str]) -> None:
|
||||
"""Setzt die bekannten Datei-Hashes für inkrementelle Verarbeitung.
|
||||
|
||||
Args:
|
||||
hashes: Mapping von Dateipfad (str) zu Content-Hash.
|
||||
Dateien mit unverändertem Hash werden übersprungen.
|
||||
"""
|
||||
self._known_hashes = dict(hashes)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Implementierung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_directory(self, config: SourceConfig | None) -> Path:
|
||||
"""Löst das Quellverzeichnis aus config oder self._directory auf.
|
||||
|
||||
Args:
|
||||
config: Optionale SourceConfig mit params["directory"].
|
||||
|
||||
Returns:
|
||||
Path zum Quellverzeichnis.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn weder config noch self._directory gesetzt ist.
|
||||
"""
|
||||
if config is not None:
|
||||
dir_param = config.params.get("directory", "")
|
||||
if dir_param:
|
||||
return Path(dir_param)
|
||||
|
||||
if self._directory is not None:
|
||||
return self._directory
|
||||
|
||||
raise ValueError(
|
||||
"Kein Verzeichnis angegeben: "
|
||||
"weder config.params['directory'] noch directory-Parameter gesetzt."
|
||||
)
|
||||
|
||||
def _do_extract(
|
||||
self, directory: Path
|
||||
) -> tuple[list[Artifact], list[SourceError], int]:
|
||||
"""Führt die eigentliche Extraktion durch.
|
||||
|
||||
Args:
|
||||
directory: Das zu scannende Verzeichnis.
|
||||
|
||||
Returns:
|
||||
Tuple von (artifacts, errors, skipped_count).
|
||||
"""
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
skipped = 0
|
||||
|
||||
# Verzeichnis-Validierung
|
||||
if not directory.exists():
|
||||
logger.warning(
|
||||
"Quellverzeichnis existiert nicht: %s", directory
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
file_path=str(self.directory),
|
||||
error=f"Verzeichnis existiert nicht: {self.directory}",
|
||||
file_path=str(directory),
|
||||
error=f"Verzeichnis existiert nicht: {directory}",
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
return artifacts
|
||||
return artifacts, errors, skipped
|
||||
|
||||
if not self.directory.is_dir():
|
||||
if not directory.is_dir():
|
||||
logger.warning(
|
||||
"Quellpfad ist kein Verzeichnis: %s", self.directory
|
||||
"Quellpfad ist kein Verzeichnis: %s", directory
|
||||
)
|
||||
self.errors.append(
|
||||
errors.append(
|
||||
SourceError(
|
||||
file_path=str(self.directory),
|
||||
error=f"Pfad ist kein Verzeichnis: {self.directory}",
|
||||
file_path=str(directory),
|
||||
error=f"Pfad ist kein Verzeichnis: {directory}",
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
return artifacts
|
||||
return artifacts, errors, skipped
|
||||
|
||||
md_files = sorted(self.directory.rglob("*.md"))
|
||||
# Alle unterstützten Dateien finden
|
||||
all_files = sorted(
|
||||
f
|
||||
for f in directory.rglob("*")
|
||||
if f.is_file() and f.suffix.lower() in _SUPPORTED_EXTENSIONS
|
||||
)
|
||||
|
||||
for md_file in md_files:
|
||||
try:
|
||||
artifact = parse_frontmatter(md_file)
|
||||
for file_path in all_files:
|
||||
# Inkrementelle Verarbeitung: Hash-Check
|
||||
if self._known_hashes:
|
||||
file_key = str(file_path)
|
||||
current_hash = _compute_file_hash(file_path)
|
||||
known_hash = self._known_hashes.get(file_key)
|
||||
if known_hash == current_hash:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Datei verarbeiten
|
||||
artifact = self._extract_file(file_path, errors)
|
||||
if artifact is not None:
|
||||
artifacts.append(artifact)
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
"Überspringe Datei ohne gültiges Frontmatter: %s – %s",
|
||||
md_file,
|
||||
e,
|
||||
)
|
||||
self.errors.append(
|
||||
SourceError(
|
||||
file_path=str(md_file),
|
||||
error=str(e),
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Lesen der Datei: %s – %s", md_file, e
|
||||
)
|
||||
self.errors.append(
|
||||
SourceError(
|
||||
file_path=str(md_file),
|
||||
error=str(e),
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
|
||||
return artifacts
|
||||
return artifacts, errors, skipped
|
||||
|
||||
def _extract_file(
|
||||
self, file_path: Path, errors: list[SourceError]
|
||||
) -> Artifact | None:
|
||||
"""Extrahiert ein einzelnes Artefakt aus einer Datei.
|
||||
|
||||
.md-Dateien: Parsed mit parse_frontmatter (erwartet Frontmatter).
|
||||
.txt-Dateien: Liest als Plain-Text und erstellt minimales Artefakt.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
errors: Fehlerliste zum Ergänzen bei Problemen.
|
||||
|
||||
Returns:
|
||||
Artifact oder None bei Fehler.
|
||||
"""
|
||||
try:
|
||||
if file_path.suffix.lower() == ".txt":
|
||||
return self._extract_txt_file(file_path)
|
||||
else:
|
||||
return parse_frontmatter(file_path)
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
"Überspringe Datei ohne gültiges Frontmatter: %s – %s",
|
||||
file_path,
|
||||
e,
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
file_path=str(file_path),
|
||||
error=str(e),
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
return None
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Lesen der Datei: %s – %s", file_path, e
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
file_path=str(file_path),
|
||||
error=str(e),
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
return None
|
||||
|
||||
def _extract_txt_file(self, file_path: Path) -> Artifact:
|
||||
"""Extrahiert ein Artefakt aus einer .txt-Datei.
|
||||
|
||||
Erstellt ein minimales Artefakt ohne Frontmatter – Titel wird
|
||||
aus dem Dateinamen abgeleitet.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur .txt-Datei.
|
||||
|
||||
Returns:
|
||||
Artifact mit minimalem Metadata.
|
||||
"""
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
title = file_path.stem.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type="note",
|
||||
title=title,
|
||||
source={"type": "file", "file": str(file_path)},
|
||||
)
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=file_path,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Legacy-Kompatibilität: Attribut-Zugriff
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def directory(self) -> Path | None:
|
||||
"""Zugriff auf das konfigurierte Verzeichnis (Legacy-Kompatibilität)."""
|
||||
return self._directory
|
||||
|
||||
@directory.setter
|
||||
def directory(self, value: Path) -> None:
|
||||
"""Setzt das Verzeichnis (Legacy-Kompatibilität)."""
|
||||
self._directory = value
|
||||
|
||||
@@ -0,0 +1,407 @@
|
||||
"""PDF- und OCR-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Implementiert die Extraktion von Text aus PDF-Dateien (eingebetteter Text
|
||||
und OCR-Fallback für gescannte Seiten) sowie die direkte OCR-Verarbeitung
|
||||
von Bilddateien (PNG, JPG, JPEG).
|
||||
|
||||
OCR-Strategie (Priorität):
|
||||
1. Kiro Vision – Bilder werden an den AI-Agenten gesendet (primär)
|
||||
2. Tesseract (optional, spätere Alternative für CI/Batch)
|
||||
|
||||
Requirements: 4.2, 4.3, 4.4, 4.5, 4.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata, compute_content_hash
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Unterstützte Dateierweiterungen
|
||||
PDF_EXTENSIONS = {".pdf"}
|
||||
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg"}
|
||||
SUPPORTED_EXTENSIONS = PDF_EXTENSIONS | IMAGE_EXTENSIONS
|
||||
|
||||
# OCR-Sprachkonfiguration
|
||||
OCR_LANGUAGES = "deu+eng"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDF-Textextraktion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_pdf_text(file_path: Path) -> tuple[str, bool]:
|
||||
"""Extrahiert Text aus einer PDF-Datei.
|
||||
|
||||
Versucht eingebetteten Text via pypdf zu extrahieren. Falls Seiten
|
||||
keinen Text enthalten (gescannte PDFs), wird ein OCR-Fallback benötigt.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur PDF-Datei.
|
||||
|
||||
Returns:
|
||||
Tuple (extrahierter_text, needs_ocr).
|
||||
needs_ocr ist True wenn mindestens eine Seite keinen Text enthält.
|
||||
|
||||
Raises:
|
||||
ImportError: Wenn keine PDF-Bibliothek verfügbar ist.
|
||||
OSError: Bei Dateizugriffsfehlern.
|
||||
"""
|
||||
try:
|
||||
from pypdf import PdfReader # type: ignore[import-untyped]
|
||||
except ImportError:
|
||||
try:
|
||||
from PyPDF2 import PdfReader # type: ignore[import-untyped, no-redef]
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"Keine PDF-Bibliothek verfügbar. "
|
||||
"Installiere 'pypdf' oder 'PyPDF2': pip install pypdf"
|
||||
)
|
||||
|
||||
reader = PdfReader(str(file_path))
|
||||
pages_text: list[str] = []
|
||||
pages_without_text: list[int] = []
|
||||
|
||||
for i, page in enumerate(reader.pages):
|
||||
text = page.extract_text() or ""
|
||||
text = text.strip()
|
||||
if text:
|
||||
pages_text.append(text)
|
||||
else:
|
||||
pages_without_text.append(i)
|
||||
|
||||
full_text = "\n\n".join(pages_text)
|
||||
needs_ocr = len(pages_without_text) > 0
|
||||
|
||||
if needs_ocr:
|
||||
logger.info(
|
||||
"PDF %s: %d von %d Seiten ohne Text (OCR benötigt für Seiten: %s)",
|
||||
file_path.name,
|
||||
len(pages_without_text),
|
||||
len(reader.pages),
|
||||
pages_without_text,
|
||||
)
|
||||
|
||||
return full_text, needs_ocr
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OCR-Engine (Kiro Vision als primärer Provider)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _perform_ocr(file_path: Path, languages: str = OCR_LANGUAGES) -> tuple[str, bool]:
|
||||
"""Führt OCR auf einer Bild- oder PDF-Datei durch.
|
||||
|
||||
Primärer Provider: Kiro Vision (über kiro_client-Integration).
|
||||
Fallback: Stub, der ocr_failed=True zurückgibt wenn Kiro nicht verfügbar.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Bild- oder PDF-Datei.
|
||||
languages: Sprachkonfiguration (default: deu+eng).
|
||||
|
||||
Returns:
|
||||
Tuple (extrahierter_text, ocr_failed).
|
||||
ocr_failed ist True wenn OCR fehlgeschlagen ist.
|
||||
"""
|
||||
# Versuche Kiro Vision Client
|
||||
try:
|
||||
from monorepo.knowledge.integrations.kiro_client import ( # type: ignore[import-untyped]
|
||||
KiroClient,
|
||||
)
|
||||
|
||||
client = KiroClient()
|
||||
text = client.ocr(file_path, languages=languages)
|
||||
if text and text.strip():
|
||||
return text.strip(), False
|
||||
else:
|
||||
logger.warning(
|
||||
"Kiro Vision lieferte keinen Text für: %s", file_path
|
||||
)
|
||||
return "", True
|
||||
except ImportError:
|
||||
logger.info(
|
||||
"kiro_client nicht verfügbar, versuche Tesseract-Fallback für: %s",
|
||||
file_path,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Kiro Vision OCR fehlgeschlagen für %s: %s", file_path, e
|
||||
)
|
||||
|
||||
# Fallback: Tesseract (optional)
|
||||
try:
|
||||
import pytesseract # type: ignore[import-untyped]
|
||||
from PIL import Image # type: ignore[import-untyped]
|
||||
|
||||
if file_path.suffix.lower() in IMAGE_EXTENSIONS:
|
||||
img = Image.open(file_path)
|
||||
text = pytesseract.image_to_string(img, lang=languages.replace("+", "+"))
|
||||
if text and text.strip():
|
||||
return text.strip(), False
|
||||
else:
|
||||
# Für PDFs: Tesseract kann nur Bilder verarbeiten
|
||||
logger.info(
|
||||
"Tesseract-OCR nur für Bilddateien verfügbar, nicht für PDF-Seiten: %s",
|
||||
file_path,
|
||||
)
|
||||
return "", True
|
||||
except ImportError:
|
||||
logger.info(
|
||||
"Weder kiro_client noch pytesseract verfügbar für OCR: %s",
|
||||
file_path,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Tesseract OCR fehlgeschlagen für %s: %s", file_path, e)
|
||||
|
||||
return "", True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDFSource Strategie
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PDFSource(SourceStrategy):
|
||||
"""Quellstrategie für PDF- und Bilddateien.
|
||||
|
||||
Verarbeitet:
|
||||
- PDF-Dateien: Eingebetteten Text extrahieren, OCR-Fallback für gescannte Seiten
|
||||
- Bilddateien (PNG, JPG, JPEG): Direkt an OCR weiterleiten
|
||||
|
||||
Setzt `ocr_failed: true` im Artefakt-Frontmatter falls OCR fehlschlägt.
|
||||
Behält Referenz auf Originaldatei in `source.file`.
|
||||
"""
|
||||
|
||||
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
|
||||
"""Extrahiert Artefakte aus PDF- und Bilddateien.
|
||||
|
||||
Args:
|
||||
config: SourceConfig mit params["path"] oder params["directory"].
|
||||
context: Der Zielkontext (bahn, dhive, privat).
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
result = ExtractionResult()
|
||||
|
||||
# Pfad aus Konfiguration ermitteln
|
||||
source_path = self._resolve_source_path(config)
|
||||
if source_path is None:
|
||||
result.errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message="Kein gültiger Pfad in config.params['path'] oder config.params['directory']",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
# Dateien sammeln
|
||||
files = self._collect_files(source_path)
|
||||
if not files:
|
||||
logger.info("Keine unterstützten Dateien gefunden in: %s", source_path)
|
||||
return result
|
||||
|
||||
# Dateien verarbeiten
|
||||
for file_path in files:
|
||||
try:
|
||||
artifact = self._process_file(file_path, config, context)
|
||||
if artifact is not None:
|
||||
result.artifacts.append(artifact)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Fehler bei der Verarbeitung von %s: %s", file_path, e
|
||||
)
|
||||
result.errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler bei {file_path.name}: {e}",
|
||||
timestamp=datetime.now().isoformat(),
|
||||
retry=True,
|
||||
artifact_path=str(file_path),
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""PDF-Source unterstützt inkrementelle Verarbeitung via Content-Hash."""
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Hilfsmethoden
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_source_path(self, config: SourceConfig) -> Path | None:
|
||||
"""Ermittelt den Quellpfad aus der Konfiguration."""
|
||||
path_str = config.params.get("path") or config.params.get("directory") or ""
|
||||
if not path_str:
|
||||
return None
|
||||
source_path = Path(path_str)
|
||||
if not source_path.exists():
|
||||
return None
|
||||
return source_path
|
||||
|
||||
def _collect_files(self, source_path: Path) -> list[Path]:
|
||||
"""Sammelt alle unterstützten Dateien aus dem Quellpfad."""
|
||||
if source_path.is_file():
|
||||
if source_path.suffix.lower() in SUPPORTED_EXTENSIONS:
|
||||
return [source_path]
|
||||
return []
|
||||
|
||||
# Verzeichnis rekursiv durchsuchen
|
||||
files: list[Path] = []
|
||||
for ext in SUPPORTED_EXTENSIONS:
|
||||
files.extend(source_path.rglob(f"*{ext}"))
|
||||
return sorted(files)
|
||||
|
||||
def _process_file(
|
||||
self, file_path: Path, config: SourceConfig, context: str
|
||||
) -> Artifact | None:
|
||||
"""Verarbeitet eine einzelne PDF- oder Bilddatei.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
config: Die Quellkonfiguration.
|
||||
context: Der Zielkontext.
|
||||
|
||||
Returns:
|
||||
Ein Artifact oder None bei Fehler.
|
||||
"""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix in PDF_EXTENSIONS:
|
||||
return self._process_pdf(file_path, config, context)
|
||||
elif suffix in IMAGE_EXTENSIONS:
|
||||
return self._process_image(file_path, config, context)
|
||||
else:
|
||||
return None
|
||||
|
||||
def _process_pdf(
|
||||
self, file_path: Path, config: SourceConfig, context: str
|
||||
) -> Artifact:
|
||||
"""Verarbeitet eine PDF-Datei.
|
||||
|
||||
1. Versucht eingebetteten Text zu extrahieren
|
||||
2. Falls Seiten ohne Text: OCR-Fallback
|
||||
3. Setzt ocr_failed bei Fehlschlag
|
||||
"""
|
||||
ocr_failed = False
|
||||
text = ""
|
||||
|
||||
try:
|
||||
embedded_text, needs_ocr = _extract_pdf_text(file_path)
|
||||
text = embedded_text
|
||||
|
||||
if needs_ocr and not embedded_text:
|
||||
# Komplett gescanntes PDF – OCR für das gesamte Dokument
|
||||
ocr_text, ocr_failed = _perform_ocr(file_path)
|
||||
if ocr_text:
|
||||
text = ocr_text
|
||||
elif needs_ocr and embedded_text:
|
||||
# Teilweise gescannt – Text vorhanden, OCR nur für fehlende Seiten
|
||||
# Für jetzt verwenden wir den vorhandenen Text
|
||||
logger.info(
|
||||
"PDF %s: Eingebetteter Text vorhanden, gescannte Seiten übersprungen",
|
||||
file_path.name,
|
||||
)
|
||||
except ImportError as e:
|
||||
logger.warning("PDF-Bibliothek nicht verfügbar: %s", e)
|
||||
# Versuche OCR als Fallback
|
||||
ocr_text, ocr_failed = _perform_ocr(file_path)
|
||||
text = ocr_text
|
||||
|
||||
return self._create_artifact(
|
||||
file_path=file_path,
|
||||
text=text,
|
||||
context=context,
|
||||
source_type="pdf",
|
||||
ocr_failed=ocr_failed,
|
||||
)
|
||||
|
||||
def _process_image(
|
||||
self, file_path: Path, config: SourceConfig, context: str
|
||||
) -> Artifact:
|
||||
"""Verarbeitet eine Bilddatei via OCR.
|
||||
|
||||
Bilddateien werden direkt an die OCR-Engine weitergeleitet.
|
||||
"""
|
||||
text, ocr_failed = _perform_ocr(file_path)
|
||||
|
||||
return self._create_artifact(
|
||||
file_path=file_path,
|
||||
text=text,
|
||||
context=context,
|
||||
source_type="image",
|
||||
ocr_failed=ocr_failed,
|
||||
)
|
||||
|
||||
def _create_artifact(
|
||||
self,
|
||||
file_path: Path,
|
||||
text: str,
|
||||
context: str,
|
||||
source_type: str,
|
||||
ocr_failed: bool,
|
||||
) -> Artifact:
|
||||
"""Erstellt ein Artifact aus den extrahierten Daten.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Originaldatei.
|
||||
text: Der extrahierte Text.
|
||||
context: Der Zielkontext.
|
||||
source_type: "pdf" oder "image".
|
||||
ocr_failed: Ob OCR fehlgeschlagen ist.
|
||||
|
||||
Returns:
|
||||
Ein vollständiges Artifact.
|
||||
"""
|
||||
# Titel aus Dateinamen ableiten
|
||||
title = file_path.stem.replace("-", " ").replace("_", " ").strip()
|
||||
if not title:
|
||||
title = file_path.name
|
||||
|
||||
# Content-Hash berechnen
|
||||
content_hash = compute_content_hash(text) if text else ""
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type="reference",
|
||||
title=title,
|
||||
tags=[source_type],
|
||||
source_context=context,
|
||||
created=date.today(),
|
||||
updated=date.today(),
|
||||
shareable=False,
|
||||
content_hash=content_hash,
|
||||
source={
|
||||
"type": source_type,
|
||||
"file": str(file_path),
|
||||
},
|
||||
category="references",
|
||||
ocr_failed=ocr_failed,
|
||||
)
|
||||
|
||||
# Markdown-Inhalt zusammenbauen
|
||||
content = f"# {title}\n\n{text}\n" if text else f"# {title}\n\n"
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=file_path,
|
||||
)
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Wissensdatenbank-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Adapter für `bahn/wissensdatenbank/output/` – liest Markdown-Artefakte
|
||||
als Read-Only-Referenzen in den bahn/knowledge/ YAML-Index ein, ohne
|
||||
die Originaldateien zu verändern oder dem Google OKF-Format aufzuzwingen.
|
||||
|
||||
Requirements: 13.1, 13.2, 13.3, 13.4, 13.5
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import (
|
||||
ExtractionResult,
|
||||
SourceConfig,
|
||||
SourceError,
|
||||
SourceStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["WissensdatenbankSource"]
|
||||
|
||||
|
||||
# Unterstützte Dateiendungen in der Wissensdatenbank
|
||||
_SUPPORTED_EXTENSIONS = {".md", ".txt"}
|
||||
|
||||
|
||||
def _compute_file_hash(file_path: Path) -> str:
|
||||
"""Berechnet den SHA-256-Hash des Dateiinhalts.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Datei.
|
||||
|
||||
Returns:
|
||||
Hash-String im Format "sha256:<hex>".
|
||||
"""
|
||||
content = file_path.read_bytes()
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
class WissensdatenbankSource(SourceStrategy):
|
||||
"""Quellstrategie: Read-Only-Referenzen aus bahn/wissensdatenbank/output/.
|
||||
|
||||
Scannt das Wissensdatenbank-Output-Verzeichnis nach Markdown-Dateien
|
||||
und erstellt Artifact-Objekte mit `source: wissensdatenbank`-Metadaten.
|
||||
|
||||
**Kernverhalten:**
|
||||
- Originaldateien werden NICHT modifiziert (Read-Only-Referenz)
|
||||
- Kein Google-OKF-Reformat (kein Splitting, keine index.md-Generierung)
|
||||
- Artefakte werden als Referenzen im Index geführt (source: wissensdatenbank)
|
||||
- Inkrementelle Verarbeitung via Content-Hash
|
||||
|
||||
Requirements: 13.1, 13.2, 13.3, 13.4, 13.5
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialisiert die WissensdatenbankSource."""
|
||||
self._known_hashes: dict[str, str] = {}
|
||||
|
||||
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
|
||||
"""Extrahiert Read-Only-Referenzen aus dem Wissensdatenbank-Output.
|
||||
|
||||
Scannt `params["output_path"]` rekursiv nach .md/.txt-Dateien und
|
||||
erstellt Artifact-Objekte ohne die Originaldateien zu verändern.
|
||||
|
||||
Args:
|
||||
config: Die Quellkonfiguration mit `params["output_path"]`.
|
||||
context: Der Zielkontext (typischerweise "bahn").
|
||||
|
||||
Returns:
|
||||
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
|
||||
"""
|
||||
output_path_str = config.params.get("output_path", "")
|
||||
if not output_path_str:
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message="Kein 'output_path' in params konfiguriert.",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
output_path = Path(output_path_str)
|
||||
|
||||
if not output_path.exists():
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="connection",
|
||||
message=f"Wissensdatenbank-Verzeichnis existiert nicht: {output_path}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
if not output_path.is_dir():
|
||||
return ExtractionResult(
|
||||
errors=[
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Pfad ist kein Verzeichnis: {output_path}",
|
||||
timestamp="",
|
||||
retry=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
artifacts: list[Artifact] = []
|
||||
errors: list[SourceError] = []
|
||||
skipped = 0
|
||||
|
||||
# Alle unterstützten Dateien finden (rekursiv)
|
||||
all_files = sorted(
|
||||
f
|
||||
for f in output_path.rglob("*")
|
||||
if f.is_file() and f.suffix.lower() in _SUPPORTED_EXTENSIONS
|
||||
)
|
||||
|
||||
for file_path in all_files:
|
||||
# Inkrementelle Verarbeitung: Hash-Check
|
||||
file_key = str(file_path)
|
||||
try:
|
||||
current_hash = _compute_file_hash(file_path)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Lesen der Datei: %s – %s", file_path, e
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler beim Lesen: {file_path} – {e}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
artifact_path=file_key,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
known_hash = self._known_hashes.get(file_key)
|
||||
if known_hash == current_hash:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Datei lesen und als Read-Only-Referenz erstellen
|
||||
try:
|
||||
artifact = self._create_reference_artifact(
|
||||
file_path, current_hash, context
|
||||
)
|
||||
artifacts.append(artifact)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Verarbeiten: %s – %s", file_path, e
|
||||
)
|
||||
errors.append(
|
||||
SourceError(
|
||||
source_name=config.name,
|
||||
error_type="parse",
|
||||
message=f"Fehler beim Verarbeiten: {file_path} – {e}",
|
||||
timestamp="",
|
||||
retry=True,
|
||||
artifact_path=file_key,
|
||||
)
|
||||
)
|
||||
|
||||
return ExtractionResult(
|
||||
artifacts=artifacts,
|
||||
errors=errors,
|
||||
skipped=skipped,
|
||||
)
|
||||
|
||||
def supports_incremental(self) -> bool:
|
||||
"""Wissensdatenbank unterstützt inkrementelle Verarbeitung via Hash.
|
||||
|
||||
Returns:
|
||||
True – inkrementelle Verarbeitung wird unterstützt.
|
||||
"""
|
||||
return True
|
||||
|
||||
def set_known_hashes(self, hashes: dict[str, str]) -> None:
|
||||
"""Setzt die bekannten Datei-Hashes für inkrementelle Verarbeitung.
|
||||
|
||||
Args:
|
||||
hashes: Mapping von Dateipfad (str) zu Content-Hash.
|
||||
Dateien mit unverändertem Hash werden übersprungen.
|
||||
"""
|
||||
self._known_hashes = dict(hashes)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Implementierung
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _create_reference_artifact(
|
||||
self, file_path: Path, content_hash: str, context: str
|
||||
) -> Artifact:
|
||||
"""Erstellt ein Read-Only-Referenz-Artefakt aus einer Wissensdatenbank-Datei.
|
||||
|
||||
Die Originaldatei wird gelesen aber NICHT verändert. Das Artefakt
|
||||
erhält `source: {"type": "wissensdatenbank", "path": <original_path>}`
|
||||
als Metadaten.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Originaldatei in wissensdatenbank/output/.
|
||||
content_hash: Bereits berechneter SHA-256-Hash.
|
||||
context: Der Zielkontext (typischerweise "bahn").
|
||||
|
||||
Returns:
|
||||
Artifact mit Read-Only-Referenz-Metadaten.
|
||||
"""
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
|
||||
# Titel aus Dateiname ableiten (ersetzt Bindestriche/Underscores)
|
||||
title = file_path.stem.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
metadata = ArtifactMetadata(
|
||||
type="reference",
|
||||
title=title,
|
||||
source_context=context,
|
||||
source={"type": "wissensdatenbank", "path": str(file_path)},
|
||||
category="references",
|
||||
content_hash=content_hash,
|
||||
created=date.today(),
|
||||
)
|
||||
|
||||
return Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=file_path,
|
||||
)
|
||||
@@ -4,6 +4,8 @@ Basiert auf der DB-Wissensdatenbank-ETL-Architektur und bietet:
|
||||
- Ingestion von Wissensartefakten aus verschiedenen Quellen
|
||||
- Progressive-Disclosure-Index (kompakte YAML-Metadaten, Schicht 1)
|
||||
- Volltextsuche mit Scope-basierter Zugriffskontrolle und Relevanz-Sortierung
|
||||
- Kontextübergreifende Suche über alle Knowledge-Folder _index.yaml-Dateien
|
||||
- Agent-Context-Injection: max. 10 relevante Artefakt-Pfade pro Anfrage
|
||||
- Graph-Verknüpfungen zwischen Artefakten
|
||||
- Timeout-Handling für Suchanfragen (5 Sekunden)
|
||||
"""
|
||||
@@ -26,6 +28,12 @@ logger = logging.getLogger(__name__)
|
||||
# Default-Timeout für Suchanfragen in Sekunden
|
||||
SEARCH_TIMEOUT_SECONDS: float = 5.0
|
||||
|
||||
# Max. Artefakt-Pfade für Agent-Context-Injection
|
||||
MAX_CONTEXT_INJECTION_RESULTS: int = 10
|
||||
|
||||
# Bekannte Kontext-Knowledge-Folder (relativ zum Monorepo-Root)
|
||||
CONTEXT_KNOWLEDGE_FOLDERS: list[str] = ["bahn", "dhive", "privat"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
@@ -48,6 +56,24 @@ class SearchResult:
|
||||
partial_results: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContextInjectionResult:
|
||||
"""Ergebnis der Agent-Context-Injection.
|
||||
|
||||
Enthält die relevantesten Artefakt-Pfade für den Agent-Prompt,
|
||||
sortiert nach Relevanz-Score. Maximal MAX_CONTEXT_INJECTION_RESULTS Einträge.
|
||||
|
||||
Attributes:
|
||||
paths: Liste der relevantesten Artefakt-Pfade (max. 10).
|
||||
entries: Die zugehörigen SearchResult-Objekte (Progressive Disclosure Schicht 1).
|
||||
partial_results: True wenn die Suche wegen Timeout abgebrochen wurde.
|
||||
"""
|
||||
|
||||
paths: list[str] = field(default_factory=list)
|
||||
entries: list[SearchResult] = field(default_factory=list)
|
||||
partial_results: bool = False
|
||||
|
||||
|
||||
def _compute_relevance(query_lower: str, entry: IndexEntry) -> float:
|
||||
"""Berechnet den Relevanz-Score für einen Index-Eintrag.
|
||||
|
||||
@@ -115,6 +141,11 @@ class KnowledgeStore:
|
||||
- Schicht 2 (Dokument): Vollständige Markdown-Datei mit YAML-Frontmatter
|
||||
|
||||
Agenten lesen zuerst den Index und laden nur bei Bedarf einzelne Dokumente.
|
||||
|
||||
Kontextübergreifende Suche:
|
||||
- Durchsucht alle Knowledge-Folder _index.yaml-Dateien (bahn, dhive, privat)
|
||||
- Respektiert ctx-guard-Sicherheitsgrenzen (nur autorisierte Scopes)
|
||||
- Agent-Context-Injection: max. 10 relevante Pfade pro Anfrage
|
||||
"""
|
||||
|
||||
INDEX_FILENAME = "_index.yaml"
|
||||
@@ -124,7 +155,7 @@ class KnowledgeStore:
|
||||
|
||||
Args:
|
||||
base_path: Basispfad des Wissensspeichers
|
||||
(z.B. shared/knowledge-store/).
|
||||
(z.B. shared/knowledge-store/ oder Monorepo-Root).
|
||||
scope_config: Konfiguration des Scope-Mappings
|
||||
(Arbeitskontext → Scope).
|
||||
"""
|
||||
@@ -350,3 +381,222 @@ class KnowledgeStore:
|
||||
|
||||
# Datei zurückschreiben (mit aktualisiertem Frontmatter)
|
||||
write_artifact(artifact, self.base_path)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Kontextübergreifende Suche über Knowledge-Folder
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def _load_knowledge_folder_indices(self) -> list[YAMLIndex]:
|
||||
"""Lädt alle _index.yaml-Dateien aus den Knowledge-Foldern.
|
||||
|
||||
Durchsucht die bekannten Kontext-Knowledge-Folder (bahn, dhive, privat)
|
||||
und lädt deren _index.yaml als YAMLIndex-Instanzen.
|
||||
|
||||
Returns:
|
||||
Liste geladener YAMLIndex-Instanzen (nur existierende Dateien).
|
||||
"""
|
||||
indices: list[YAMLIndex] = []
|
||||
|
||||
for context_name in CONTEXT_KNOWLEDGE_FOLDERS:
|
||||
index_path = self.base_path / context_name / "knowledge" / self.INDEX_FILENAME
|
||||
if index_path.exists():
|
||||
try:
|
||||
idx = YAMLIndex(index_path)
|
||||
idx.load()
|
||||
indices.append(idx)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Konnte Knowledge-Folder-Index nicht laden: %s (%s)",
|
||||
index_path,
|
||||
e,
|
||||
)
|
||||
|
||||
return indices
|
||||
|
||||
def search_cross_context(
|
||||
self,
|
||||
query: str,
|
||||
allowed_scopes: list[str],
|
||||
timeout: float | None = None,
|
||||
) -> list[SearchResult]:
|
||||
"""Kontextübergreifende Suche über alle Knowledge-Folder _index.yaml-Dateien.
|
||||
|
||||
Durchsucht sowohl den eigenen Index als auch alle Kontext-Knowledge-Folder
|
||||
(_index.yaml in bahn/knowledge/, dhive/knowledge/, privat/knowledge/).
|
||||
Respektiert ctx-guard-Sicherheitsgrenzen: nur Artefakte aus autorisierten
|
||||
Scopes werden zurückgegeben.
|
||||
|
||||
Progressive Disclosure (Schicht 1): Die Suche operiert ausschließlich
|
||||
auf dem YAML-Index (kompakte Metadaten). Vollständige Dokumente werden
|
||||
nur bei explizitem Nachladen über den Pfad bereitgestellt.
|
||||
|
||||
Args:
|
||||
query: Suchbegriff für die Volltextsuche.
|
||||
allowed_scopes: Liste der Scopes, aus denen Ergebnisse
|
||||
geliefert werden dürfen (ctx-guard-Autorisierung).
|
||||
timeout: Timeout in Sekunden (None = SEARCH_TIMEOUT_SECONDS).
|
||||
|
||||
Returns:
|
||||
Liste von SearchResult-Objekten, sortiert nach Relevanz
|
||||
(höchster Score zuerst). Enthält Ergebnisse aus dem eigenen
|
||||
Index UND allen Knowledge-Folder-Indices.
|
||||
|
||||
Note:
|
||||
Requirements: 11.1, 11.3, 11.5
|
||||
"""
|
||||
if not query:
|
||||
return []
|
||||
|
||||
effective_timeout = timeout if timeout is not None else SEARCH_TIMEOUT_SECONDS
|
||||
query_lower = query.lower()
|
||||
results: list[SearchResult] = []
|
||||
seen_ids: set[str] = set()
|
||||
timed_out = False
|
||||
start_time = time.monotonic()
|
||||
|
||||
# 1. Eigenen Index durchsuchen (shared KnowledgeStore)
|
||||
for entry in self.index.entries.values():
|
||||
elapsed = time.monotonic() - start_time
|
||||
if elapsed >= effective_timeout:
|
||||
timed_out = True
|
||||
break
|
||||
|
||||
if entry.scope not in allowed_scopes:
|
||||
continue
|
||||
|
||||
searchable_text = " ".join([
|
||||
entry.title, entry.id, entry.summary, " ".join(entry.tags),
|
||||
]).lower()
|
||||
|
||||
if query_lower in searchable_text:
|
||||
relevance = _compute_relevance(query_lower, entry)
|
||||
results.append(SearchResult(
|
||||
entry=entry, relevance_score=relevance, partial_results=False,
|
||||
))
|
||||
seen_ids.add(entry.id)
|
||||
|
||||
# 2. Knowledge-Folder-Indices durchsuchen
|
||||
if not timed_out:
|
||||
folder_indices = self._load_knowledge_folder_indices()
|
||||
|
||||
for folder_index in folder_indices:
|
||||
if timed_out:
|
||||
break
|
||||
|
||||
for entry in folder_index.entries.values():
|
||||
elapsed = time.monotonic() - start_time
|
||||
if elapsed >= effective_timeout:
|
||||
timed_out = True
|
||||
break
|
||||
|
||||
# Scope-Filter: Nur autorisierte Scopes
|
||||
if entry.scope not in allowed_scopes:
|
||||
continue
|
||||
|
||||
# Deduplizierung: Einträge die bereits im eigenen Index sind
|
||||
if entry.id in seen_ids:
|
||||
continue
|
||||
|
||||
searchable_text = " ".join([
|
||||
entry.title, entry.id, entry.summary, " ".join(entry.tags),
|
||||
]).lower()
|
||||
|
||||
if query_lower in searchable_text:
|
||||
relevance = _compute_relevance(query_lower, entry)
|
||||
results.append(SearchResult(
|
||||
entry=entry, relevance_score=relevance, partial_results=False,
|
||||
))
|
||||
seen_ids.add(entry.id)
|
||||
|
||||
# Bei Timeout: Alle Ergebnisse als partial markieren
|
||||
if timed_out:
|
||||
for result in results:
|
||||
result.partial_results = True
|
||||
|
||||
# Nach Relevanz sortieren (höchster Score zuerst)
|
||||
results.sort(key=lambda r: r.relevance_score, reverse=True)
|
||||
|
||||
return results
|
||||
|
||||
def get_context_injection(
|
||||
self,
|
||||
query: str,
|
||||
allowed_scopes: list[str],
|
||||
max_results: int | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> ContextInjectionResult:
|
||||
"""Agent-Context-Injection: Liefert die relevantesten Artefakt-Pfade.
|
||||
|
||||
Identifiziert die relevantesten Artefakte aus den YAML-Indices aller
|
||||
autorisierten Kontexte und liefert deren Pfade als Kontextinformation
|
||||
für den Agent-Prompt.
|
||||
|
||||
Progressive Disclosure:
|
||||
- Schicht 1: YAML-Index-Daten werden für die Relevanzprüfung verwendet.
|
||||
- Schicht 2: Nur die Pfade werden zurückgegeben – das vollständige
|
||||
Dokument wird vom Agenten bei Bedarf nachgeladen.
|
||||
|
||||
Args:
|
||||
query: Suchbegriff oder Task-Beschreibung für Relevanzprüfung.
|
||||
allowed_scopes: Autorisierte Scopes (ctx-guard-Filterung).
|
||||
max_results: Maximale Anzahl zurückgegebener Pfade
|
||||
(None = MAX_CONTEXT_INJECTION_RESULTS = 10).
|
||||
timeout: Timeout in Sekunden (None = SEARCH_TIMEOUT_SECONDS).
|
||||
|
||||
Returns:
|
||||
ContextInjectionResult mit max. 10 Pfaden, sortiert nach
|
||||
Relevanz-Score (höchster zuerst).
|
||||
|
||||
Note:
|
||||
Requirements: 11.2, 11.5, 11.6
|
||||
"""
|
||||
effective_max = max_results if max_results is not None else MAX_CONTEXT_INJECTION_RESULTS
|
||||
|
||||
# Kontextübergreifende Suche durchführen
|
||||
all_results = self.search_cross_context(query, allowed_scopes, timeout)
|
||||
|
||||
# Auf max. Ergebnisse beschränken
|
||||
top_results = all_results[:effective_max]
|
||||
|
||||
# Pfade extrahieren (nur Einträge mit gültigem Pfad)
|
||||
paths: list[str] = []
|
||||
filtered_results: list[SearchResult] = []
|
||||
for result in top_results:
|
||||
if result.entry.path:
|
||||
paths.append(result.entry.path)
|
||||
filtered_results.append(result)
|
||||
|
||||
partial = any(r.partial_results for r in top_results) if top_results else False
|
||||
|
||||
return ContextInjectionResult(
|
||||
paths=paths,
|
||||
entries=filtered_results,
|
||||
partial_results=partial,
|
||||
)
|
||||
|
||||
def load_artifact_content(self, artifact_path: str) -> str | None:
|
||||
"""Progressive Disclosure Schicht 2: Lädt den vollständigen Artefakt-Inhalt.
|
||||
|
||||
Wird vom Agenten aufgerufen, wenn nach der Relevanzprüfung (Schicht 1)
|
||||
der vollständige Dokumentinhalt benötigt wird.
|
||||
|
||||
Args:
|
||||
artifact_path: Relativer Pfad zum Artefakt (aus dem IndexEntry.path).
|
||||
|
||||
Returns:
|
||||
Der vollständige Dateiinhalt als String, oder None wenn die Datei
|
||||
nicht existiert oder nicht gelesen werden kann.
|
||||
|
||||
Note:
|
||||
Requirements: 11.3
|
||||
"""
|
||||
full_path = self.base_path / artifact_path
|
||||
if not full_path.exists():
|
||||
logger.debug("Artefakt-Datei nicht gefunden für Schicht-2-Zugriff: %s", full_path)
|
||||
return None
|
||||
|
||||
try:
|
||||
return full_path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError) as e:
|
||||
logger.warning("Konnte Artefakt nicht lesen: %s (%s)", full_path, e)
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user