feat: implement knowledge management system (spec complete, all 53 tasks done)

This commit is contained in:
2026-07-01 14:14:16 +02:00
parent e4256d3222
commit 8ac664d33d
89 changed files with 19759 additions and 72 deletions
@@ -680,6 +680,10 @@ def _build_parser() -> argparse.ArgumentParser:
# --- uninstall-hooks ---
subparsers.add_parser("uninstall-hooks", help="Git-Hooks entfernen")
# --- knowledge (Subcommand-Gruppe) ---
from monorepo.knowledge.cli import register_knowledge_subcommand
register_knowledge_subcommand(subparsers)
return parser
@@ -703,6 +707,7 @@ _COMMAND_HANDLERS = {
"fed-status": _cmd_fed_status,
"install-hooks": _cmd_install_hooks,
"uninstall-hooks": _cmd_uninstall_hooks,
"knowledge": "_knowledge", # Sentinel handled specially in main()
}
@@ -722,6 +727,13 @@ def main() -> None:
parser.print_help()
sys.exit(0)
# Special handling for knowledge subcommand (has sub-subparsers)
if args.command == "knowledge":
from monorepo.knowledge.cli import handle_knowledge_command
root = _find_monorepo_root()
exit_code = handle_knowledge_command(args, root)
sys.exit(exit_code)
handler = _COMMAND_HANDLERS.get(args.command)
if handler is None:
parser.print_help()
@@ -8,6 +8,8 @@ Stellt den KnowledgeStore, YAMLIndex, IndexEntry und das Artefakt-Modell bereit:
- Artifact/ArtifactMetadata: Vollständige Wissensartefakte mit YAML-Frontmatter
- ETLPipeline: ETL-Pipeline für inkrementelle Verarbeitung
- MarkdownSource: Quellstrategie für Markdown-Dateien
- IngestionPipeline: Neuer Orchestrator für den vollständigen Ingestion-Workflow
- QuickCapture: Schnelle Wissenserfassung über CLI
"""
from monorepo.knowledge.artifact import (
@@ -21,19 +23,29 @@ from monorepo.knowledge.artifact import (
)
from monorepo.knowledge.etl import ETLPipeline, IngestError, IngestResult
from monorepo.knowledge.index import IndexEntry, YAMLIndex
from monorepo.knowledge.ingestion.capture import QuickCapture
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline, IngestionResult
from monorepo.knowledge.sources.markdown import MarkdownSource, SourceError
from monorepo.knowledge.store import KnowledgeStore, SearchResult
from monorepo.knowledge.store import (
ContextInjectionResult,
KnowledgeStore,
SearchResult,
)
__all__ = [
"Artifact",
"ArtifactLink",
"ArtifactMetadata",
"ContextInjectionResult",
"ETLPipeline",
"IndexEntry",
"IngestError",
"IngestResult",
"IngestionPipeline",
"IngestionResult",
"KnowledgeStore",
"MarkdownSource",
"QuickCapture",
"SearchResult",
"SourceError",
"YAMLIndex",
@@ -39,6 +39,10 @@ class ArtifactMetadata:
Enthält mindestens: Typ, Titel, Tags, Quellkontext, Erstelldatum,
Aktualisierungsdatum, Teilbarkeits-Flag, Verknüpfungen, Content-Hash.
Knowledge-Management-Erweiterungen (alle optional mit Defaults für
Rückwärtskompatibilität): source, category, people, projects,
enrichment_pending, ocr_failed, fetch_failed.
"""
type: str
@@ -51,6 +55,17 @@ class ArtifactMetadata:
links: list[ArtifactLink] = field(default_factory=list)
content_hash: str = ""
# Knowledge Management extensions
source: dict[str, str] = field(default_factory=dict)
# e.g. {"type": "confluence", "url": "...", "space": "...", "file": "..."}
category: str = "" # meeting, decision, project, reference, link, inbox
people: list[str] = field(default_factory=list) # [[people/name]] refs
projects: list[str] = field(default_factory=list) # [[projects/slug]] refs
enrichment_pending: bool = False
ocr_failed: bool = False
fetch_failed: bool = False
@dataclass
class Artifact:
@@ -186,6 +201,21 @@ def parse_frontmatter(file_path: Path) -> Artifact:
raw: dict[str, Any] = yaml.safe_load(yaml_text) or {}
# Metadaten extrahieren
raw_source = raw.get("source")
source: dict[str, str] = {}
if isinstance(raw_source, dict):
source = {str(k): str(v) for k, v in raw_source.items()}
raw_people = raw.get("people")
people: list[str] = (
[str(p) for p in raw_people] if isinstance(raw_people, list) else []
)
raw_projects = raw.get("projects")
projects: list[str] = (
[str(p) for p in raw_projects] if isinstance(raw_projects, list) else []
)
metadata = ArtifactMetadata(
type=str(raw.get("type", "")),
title=str(raw.get("title", "")),
@@ -196,6 +226,13 @@ def parse_frontmatter(file_path: Path) -> Artifact:
shareable=bool(raw.get("shareable", False)),
links=_parse_links(raw.get("links")),
content_hash=str(raw.get("content_hash", "")),
source=source,
category=str(raw.get("category", "")),
people=people,
projects=projects,
enrichment_pending=bool(raw.get("enrichment_pending", False)),
ocr_failed=bool(raw.get("ocr_failed", False)),
fetch_failed=bool(raw.get("fetch_failed", False)),
)
return Artifact(metadata=metadata, content=content, file_path=file_path)
@@ -236,6 +273,28 @@ def _serialize_metadata(metadata: ArtifactMetadata) -> dict[str, Any]:
if metadata.content_hash:
data["content_hash"] = metadata.content_hash
# Knowledge Management extensions (nur wenn nicht-leer/nicht-default)
if metadata.source:
data["source"] = metadata.source
if metadata.category:
data["category"] = metadata.category
if metadata.people:
data["people"] = metadata.people
if metadata.projects:
data["projects"] = metadata.projects
if metadata.enrichment_pending:
data["enrichment_pending"] = metadata.enrichment_pending
if metadata.ocr_failed:
data["ocr_failed"] = metadata.ocr_failed
if metadata.fetch_failed:
data["fetch_failed"] = metadata.fetch_failed
return data
@@ -0,0 +1,728 @@
"""CLI-Schnittstelle für das Knowledge Management System.
Stellt das `knowledge`-Subcommand des Monorepo-CLI bereit mit folgenden
Unterkommandos:
- capture: Quick-Capture von Text, URLs oder Dateipfaden
- ingest: Pipeline-Durchlauf starten
- search: Suche über den Knowledge-Index
- status: Übersicht über Inbox-Elemente und letzte Verarbeitung
- link: Web-Link als Referenzmaterial speichern
Requirements: 15.1, 15.2, 15.3, 15.4, 15.5, 15.6, 15.7
"""
from __future__ import annotations
import argparse
import logging
import sys
from pathlib import Path
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
VALID_CONTEXTS = ("bahn", "dhive", "privat")
_HELP_TEXT = """\
Knowledge Management Wissen erfassen, verarbeiten und durchsuchen.
Unterkommandos:
capture Schnelle Wissenserfassung (Text, URL oder Dateipfad)
ingest Ingestion-Pipeline für konfigurierte Quellen starten
search Knowledge-Index durchsuchen
status Übersicht über Inbox und Artefakte
link Web-Link als Referenzmaterial speichern
Beispiele:
monorepo knowledge capture "Meeting-Notiz: API-Design besprochen" --context bahn
monorepo knowledge capture https://docs.example.com/api --tags api,referenz
monorepo knowledge ingest --context dhive --dry-run
monorepo knowledge search "API Design" --limit 5
monorepo knowledge status --context privat
monorepo knowledge link https://example.com --title "Beispiel" --tags ref
"""
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _find_monorepo_root() -> Path:
"""Find the monorepo root directory.
Delegates to the main CLI's root-finding logic.
Returns:
Absolute path to the monorepo root.
"""
from monorepo.cli import _find_monorepo_root as _main_find_root
return _main_find_root()
def _derive_context_from_cwd() -> str | None:
"""Derive context from the current working directory.
Checks if the CWD path contains a known context segment
(/bahn/, /dhive/, /privat/).
Returns:
The context string or None if it cannot be determined.
"""
cwd = Path.cwd()
path_str = str(cwd).replace("\\", "/")
for ctx in VALID_CONTEXTS:
if f"/{ctx}/" in path_str or path_str.endswith(f"/{ctx}"):
return ctx
return None
def _resolve_context(context_flag: str | None) -> str:
"""Resolve context from --context flag or current working directory.
Args:
context_flag: Explicit context from CLI flag, or None.
Returns:
The resolved context string.
Raises:
SystemExit: If context cannot be determined.
"""
if context_flag:
if context_flag not in VALID_CONTEXTS:
print(
f"Fehler: Ungültiger Kontext '{context_flag}'. "
f"Erlaubt: {', '.join(VALID_CONTEXTS)}",
file=sys.stderr,
)
sys.exit(1)
return context_flag
derived = _derive_context_from_cwd()
if derived:
return derived
print(
"Fehler: Kontext konnte nicht aus dem Arbeitsverzeichnis abgeleitet werden.\n"
"Verwende --context <bahn|dhive|privat> um den Kontext explizit anzugeben.",
file=sys.stderr,
)
sys.exit(1)
# ---------------------------------------------------------------------------
# Parser Builder
# ---------------------------------------------------------------------------
def build_knowledge_parser() -> argparse.ArgumentParser:
"""Build a standalone argument parser for the knowledge subcommand.
This can be used independently for testing or as part of the
main CLI parser registration.
Returns:
ArgumentParser configured with all knowledge subcommands.
"""
parser = argparse.ArgumentParser(
prog="knowledge",
description=_HELP_TEXT,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
subparsers = parser.add_subparsers(
dest="knowledge_command",
help="Knowledge-Unterkommandos",
)
# --- knowledge capture ---
p_capture = subparsers.add_parser(
"capture",
help="Text, URL oder Dateipfad schnell erfassen",
)
p_capture.add_argument(
"input",
help="Zu erfassender Text, URL oder Dateipfad.",
)
p_capture.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Zielkontext (Standard: aus CWD abgeleitet).",
)
p_capture.add_argument(
"--tags", "-t",
default=None,
help="Komma-getrennte Tags (z.B. 'api,meeting').",
)
# --- knowledge ingest ---
p_ingest = subparsers.add_parser(
"ingest",
help="Ingestion-Pipeline für konfigurierte Quellen starten",
)
p_ingest.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Zielkontext (Standard: aus CWD abgeleitet).",
)
p_ingest.add_argument(
"--dry-run",
action="store_true",
help="Vorschau ohne Schreiboperationen.",
)
p_ingest.add_argument(
"--verbose", "-v",
action="store_true",
help="Detaillierte Ausgabe.",
)
p_ingest.add_argument(
"--no-commit",
action="store_true",
help="Git-Commit überspringen.",
)
# --- knowledge search ---
p_search = subparsers.add_parser(
"search",
help="Knowledge-Index durchsuchen",
)
p_search.add_argument(
"query",
help="Suchbegriff.",
)
p_search.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Suche auf einen Kontext beschränken.",
)
p_search.add_argument(
"--limit", "-l",
type=int,
default=None,
help="Maximale Anzahl Ergebnisse (Standard: 20).",
)
# --- knowledge status ---
p_status = subparsers.add_parser(
"status",
help="Übersicht über Inbox-Elemente und Artefakte",
)
p_status.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Kontext anzeigen (Standard: aus CWD abgeleitet).",
)
# --- knowledge link ---
p_link = subparsers.add_parser(
"link",
help="Web-Link als Referenzmaterial speichern",
)
p_link.add_argument(
"url",
help="Die zu speichernde URL.",
)
p_link.add_argument(
"--title", "-T",
default=None,
help="Optionaler Titel für den Link.",
)
p_link.add_argument(
"--tags", "-t",
default=None,
help="Komma-getrennte Tags (z.B. 'api,referenz').",
)
p_link.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Zielkontext (Standard: aus CWD abgeleitet).",
)
return parser
# ---------------------------------------------------------------------------
# Subcommand Handlers
# ---------------------------------------------------------------------------
def _cmd_capture(args: argparse.Namespace) -> int:
"""Handle `knowledge capture <text|url|path> [--context CTX] [--tags TAG,TAG]`.
Args:
args: Parsed arguments with input, context, tags.
Returns:
Exit code (0 for success, 1 for failure).
"""
from monorepo.knowledge.ingestion.capture import QuickCapture
root = _find_monorepo_root()
context = _resolve_context(args.context)
# Parse tags
tags: list[str] = []
if args.tags:
tags = [t.strip() for t in args.tags.split(",") if t.strip()]
capture = QuickCapture(monorepo_root=root)
try:
result_path = capture.capture(
input_text=args.input,
context=context,
tags=tags,
)
print(f"✓ Erfasst: {result_path}")
return 0
except Exception as e:
print(f"Fehler beim Erfassen: {e}", file=sys.stderr)
return 1
def _cmd_ingest(args: argparse.Namespace) -> int:
"""Handle `knowledge ingest [--context CTX] [--dry-run] [--verbose] [--no-commit]`.
Args:
args: Parsed arguments with context, dry_run, verbose, no_commit.
Returns:
Exit code (0 for success, 1 for failure).
"""
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
root = _find_monorepo_root()
context = _resolve_context(args.context)
# Configure logging for verbose mode
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format="%(levelname)s: %(name)s: %(message)s",
force=True,
)
pipeline = IngestionPipeline(
monorepo_root=root,
context=context,
dry_run=args.dry_run,
no_commit=args.no_commit,
)
if args.dry_run:
print(f"[DRY RUN] Ingestion für Kontext '{context}'...")
else:
print(f"Ingestion für Kontext '{context}'...")
try:
result = pipeline.run()
except Exception as e:
print(f"Fehler bei Ingestion: {e}", file=sys.stderr)
return 1
# Print results
print(f"\n{'' * 40}")
print(f"Kontext: {result.context}")
print(f"Verarbeitet: {result.processed}")
print(f"Übersprungen: {result.skipped}")
if hasattr(result, "tasks_created") and result.tasks_created:
print(f"Tasks erstellt: {result.tasks_created}")
if hasattr(result, "commit_sha") and result.commit_sha:
print(f"Commit: {result.commit_sha[:8]}")
if result.errors:
print(f"\n{len(result.errors)} Fehler:")
for err in result.errors:
print(f" • [{err.source}] {err.message}")
return 0
def _cmd_search(args: argparse.Namespace) -> int:
"""Handle `knowledge search <query> [--context CTX] [--limit N]`.
Args:
args: Parsed arguments with query, context, limit.
Returns:
Exit code (0 for success, 1 for failure).
"""
from monorepo.knowledge.store import KnowledgeStore
from monorepo.models import ScopeConfig
root = _find_monorepo_root()
# Determine allowed scopes based on context flag
if args.context:
if args.context not in VALID_CONTEXTS:
print(
f"Fehler: Ungültiger Kontext '{args.context}'.",
file=sys.stderr,
)
return 1
allowed_scopes = [args.context]
else:
allowed_scopes = list(VALID_CONTEXTS) + ["shared"]
# Create KnowledgeStore
ks_path = root / "shared" / "knowledge-store"
scope_config = ScopeConfig(scopes={
"privat": {"scope": "privat", "paths": ["privat/"]},
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
"shared": {"scope": "shared", "paths": ["shared/"]},
})
store = KnowledgeStore(base_path=ks_path, scope_config=scope_config)
query = args.query
results = store.search_cross_context(query=query, allowed_scopes=allowed_scopes)
# Apply limit
limit = args.limit or 20
results = results[:limit]
if not results:
print(f"Keine Treffer für '{query}'.")
return 0
print(f"Treffer für '{query}' ({len(results)} Ergebnis(se)):\n")
for r in results:
scope_label = r.entry.scope if r.entry.scope else "?"
tags_str = ", ".join(r.entry.tags) if r.entry.tags else "-"
entry_type = getattr(r.entry, "type", None) or "-"
print(f" [{scope_label}] {r.entry.title}")
print(f" Typ: {entry_type}")
print(f" Tags: {tags_str}")
print(f" Pfad: {r.entry.path}")
print()
if results and results[0].partial_results:
print("⚠ Suche wegen Timeout abgebrochen möglicherweise unvollständig.")
return 0
def _cmd_status(args: argparse.Namespace) -> int:
"""Handle `knowledge status [--context CTX]`.
Args:
args: Parsed arguments with context.
Returns:
Exit code (0 for success).
"""
root = _find_monorepo_root()
context = _resolve_context(args.context)
knowledge_path = root / context / "knowledge"
inbox_path = knowledge_path / "inbox"
# Count inbox items
inbox_items: list[Path] = []
if inbox_path.exists():
inbox_items = [
f for f in inbox_path.iterdir()
if f.is_file() and f.suffix == ".md" and f.name != "index.md"
]
# Count total artifacts across all subfolders
total_artifacts = 0
subfolders = ["meetings", "decisions", "projects", "people", "references", "links"]
folder_counts: dict[str, int] = {}
for folder_name in subfolders:
folder_path = knowledge_path / folder_name
if folder_path.exists():
count = sum(
1 for f in folder_path.iterdir()
if f.is_file() and f.suffix == ".md" and f.name != "index.md"
)
folder_counts[folder_name] = count
total_artifacts += count
# Check _index.yaml
index_path = knowledge_path / "_index.yaml"
index_exists = index_path.exists()
# Check sources.yaml
sources_path = knowledge_path / "sources.yaml"
sources_exists = sources_path.exists()
# Output
print(f"Knowledge Status: {context}")
print(f"{'' * 40}")
print(f"Pfad: {knowledge_path}")
print(f"Index: {'' if index_exists else ''}")
print(f"Sources: {'' if sources_exists else ''}")
print(f"\nInbox: {len(inbox_items)} Element(e)")
print(f"Artefakte: {total_artifacts} gesamt")
if folder_counts:
print()
for folder, count in sorted(folder_counts.items()):
if count > 0:
print(f" {folder:12s} {count}")
if inbox_items:
print(f"\nInbox-Elemente:")
for item in sorted(inbox_items, key=lambda p: p.name, reverse=True)[:10]:
print(f"{item.name}")
if len(inbox_items) > 10:
print(f" ... und {len(inbox_items) - 10} weitere")
return 0
def _cmd_link(args: argparse.Namespace) -> int:
"""Handle `knowledge link <url> [--title TITLE] [--tags TAG,TAG] [--context CTX]`.
Args:
args: Parsed arguments with url, title, tags, context.
Returns:
Exit code (0 for success, 1 for failure).
"""
from monorepo.knowledge.sources.link import LinkRegistry
root = _find_monorepo_root()
context = _resolve_context(args.context)
# Parse tags
tags: list[str] = []
if args.tags:
tags = [t.strip() for t in args.tags.split(",") if t.strip()]
context_path = root / context / "knowledge"
registry = LinkRegistry(context_path)
try:
result_path = registry.save_link(
url=args.url,
title=args.title,
tags=tags,
)
print(f"✓ Link gespeichert: {result_path}")
return 0
except Exception as e:
print(f"Fehler beim Speichern des Links: {e}", file=sys.stderr)
return 1
# ---------------------------------------------------------------------------
# Main Entry Point (standalone usage)
# ---------------------------------------------------------------------------
def knowledge_main(args: argparse.Namespace) -> None:
"""Main entry point for standalone knowledge CLI usage.
Dispatches to the appropriate handler based on knowledge_command.
Calls sys.exit when no command is given.
Args:
args: Parsed argparse namespace.
"""
knowledge_cmd = getattr(args, "knowledge_command", None)
if not knowledge_cmd:
print(_HELP_TEXT)
sys.exit(0)
dispatch = {
"capture": _cmd_capture,
"ingest": _cmd_ingest,
"search": _cmd_search,
"status": _cmd_status,
"link": _cmd_link,
}
handler = dispatch.get(knowledge_cmd)
if handler is None:
print(f"Unbekanntes Unterkommando: {knowledge_cmd}", file=sys.stderr)
sys.exit(1)
exit_code = handler(args)
sys.exit(exit_code)
# ---------------------------------------------------------------------------
# Registration for main CLI integration
# ---------------------------------------------------------------------------
def register_knowledge_subcommand(subparsers: argparse._SubParsersAction) -> None:
"""Register the `knowledge` subcommand group under the main CLI parser.
This reuses the same parser structure as build_knowledge_parser but
integrates it as a subparser of the main monorepo CLI.
Args:
subparsers: The subparsers action from the main argparse parser.
"""
knowledge_parser = subparsers.add_parser(
"knowledge",
help="Wissensmanagement: Erfassen, Verarbeiten, Suchen",
description=_HELP_TEXT,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
# Sub-subparsers for knowledge commands
knowledge_sub = knowledge_parser.add_subparsers(
dest="knowledge_command",
help="Knowledge-Unterkommandos",
)
# --- knowledge capture ---
p_capture = knowledge_sub.add_parser(
"capture",
help="Text, URL oder Dateipfad schnell erfassen",
)
p_capture.add_argument(
"input",
help="Zu erfassender Text, URL oder Dateipfad.",
)
p_capture.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Zielkontext (Standard: aus CWD abgeleitet).",
)
p_capture.add_argument(
"--tags", "-t",
default=None,
help="Komma-getrennte Tags (z.B. 'api,meeting').",
)
# --- knowledge ingest ---
p_ingest = knowledge_sub.add_parser(
"ingest",
help="Ingestion-Pipeline für konfigurierte Quellen starten",
)
p_ingest.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Zielkontext (Standard: aus CWD abgeleitet).",
)
p_ingest.add_argument(
"--dry-run",
action="store_true",
help="Vorschau ohne Schreiboperationen.",
)
p_ingest.add_argument(
"--verbose", "-v",
action="store_true",
help="Detaillierte Ausgabe.",
)
p_ingest.add_argument(
"--no-commit",
action="store_true",
help="Git-Commit überspringen.",
)
# --- knowledge search ---
p_search = knowledge_sub.add_parser(
"search",
help="Knowledge-Index durchsuchen",
)
p_search.add_argument(
"query",
help="Suchbegriff.",
)
p_search.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Suche auf einen Kontext beschränken.",
)
p_search.add_argument(
"--limit", "-l",
type=int,
default=None,
help="Maximale Anzahl Ergebnisse (Standard: 20).",
)
# --- knowledge status ---
p_status = knowledge_sub.add_parser(
"status",
help="Übersicht über Inbox-Elemente und Artefakte",
)
p_status.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Kontext anzeigen (Standard: aus CWD abgeleitet).",
)
# --- knowledge link ---
p_link = knowledge_sub.add_parser(
"link",
help="Web-Link als Referenzmaterial speichern",
)
p_link.add_argument(
"url",
help="Die zu speichernde URL.",
)
p_link.add_argument(
"--title", "-T",
default=None,
help="Optionaler Titel für den Link.",
)
p_link.add_argument(
"--tags", "-t",
default=None,
help="Komma-getrennte Tags (z.B. 'api,referenz').",
)
p_link.add_argument(
"--context", "-c",
choices=VALID_CONTEXTS,
default=None,
help="Zielkontext (Standard: aus CWD abgeleitet).",
)
def handle_knowledge_command(args: argparse.Namespace, root: Path) -> int:
"""Dispatch knowledge subcommands from the main CLI.
Called from the main CLI's main() when 'knowledge' is the active command.
Uses the root provided by the main CLI instead of discovering it again.
Args:
args: Parsed arguments namespace.
root: Monorepo root path (provided by main CLI).
Returns:
Exit code (0 for success, non-zero for failure).
"""
knowledge_cmd = getattr(args, "knowledge_command", None)
if not knowledge_cmd:
print(_HELP_TEXT)
return 0
dispatch = {
"capture": _cmd_capture,
"ingest": _cmd_ingest,
"search": _cmd_search,
"status": _cmd_status,
"link": _cmd_link,
}
handler = dispatch.get(knowledge_cmd)
if handler is None:
print(f"Unbekanntes Unterkommando: {knowledge_cmd}", file=sys.stderr)
return 1
return handler(args)
@@ -0,0 +1,60 @@
"""Unified Ingestion Pipeline für den Wissensspeicher.
Orchestriert das Erfassen, Routen, Anreichern und Speichern von
Wissensartefakten aus verschiedenen Quellen in die Knowledge-Kontexte.
"""
from monorepo.knowledge.ingestion.capture import QuickCapture
from monorepo.knowledge.ingestion.config import (
ConfigValidationError,
EnrichmentSettings,
GitSettings,
OCRSettings,
PipelineSettings,
SourceConfig,
load_sources_config,
)
from monorepo.knowledge.ingestion.enrichment import (
ActionItem,
EnrichmentAgent,
EnrichmentResult,
)
from monorepo.knowledge.ingestion.git_integration import (
auto_commit,
auto_commit_capture,
generate_capture_commit_message,
generate_commit_message,
)
from monorepo.knowledge.ingestion.pipeline import (
IngestError,
IngestionPipeline,
IngestionResult,
get_source_strategy,
register_source_strategy,
)
from monorepo.knowledge.ingestion.router import ContextRouter, RoutingDecision
__all__ = [
"ActionItem",
"ConfigValidationError",
"ContextRouter",
"EnrichmentAgent",
"EnrichmentResult",
"EnrichmentSettings",
"GitSettings",
"IngestError",
"IngestionPipeline",
"IngestionResult",
"OCRSettings",
"PipelineSettings",
"QuickCapture",
"RoutingDecision",
"SourceConfig",
"auto_commit",
"auto_commit_capture",
"generate_capture_commit_message",
"generate_commit_message",
"get_source_strategy",
"load_sources_config",
"register_source_strategy",
]
@@ -0,0 +1,262 @@
"""Capture Schnelle Wissenserfassung über CLI und Integrationen.
Ermöglicht das ad-hoc Erfassen von Wissensschnipseln, Notizen und
Erkenntnissen ohne vollständigen Pipeline-Durchlauf.
Requirements: 3.1, 3.2, 3.3, 3.7, 8.3
"""
from __future__ import annotations
import re
from datetime import datetime
from pathlib import Path
from urllib.parse import urlparse
from monorepo.knowledge.ingestion.router import ContextRouter
class QuickCapture:
"""Fast knowledge capture to inbox."""
def __init__(self, monorepo_root: Path) -> None:
"""Initialize with monorepo root path.
Args:
monorepo_root: Absolute path to the monorepo root directory.
"""
self.monorepo_root = monorepo_root
self._router = ContextRouter(monorepo_root)
def capture(
self,
input_text: str,
context: str | None = None,
tags: list[str] | None = None,
) -> Path:
"""Capture text/URL/filepath to inbox. Returns path of created artifact.
Args:
input_text: The text, URL, or file path to capture.
context: Target context (bahn, dhive, privat). If None,
derived from current working directory.
tags: Optional list of tags.
Returns:
Path of the created Markdown artifact.
Raises:
ValueError: If context cannot be determined.
"""
if tags is None:
tags = []
# Determine context
if context is None:
context = self._router.determine_context(cwd=Path.cwd())
# Detect input type
input_type = self._detect_input_type(input_text)
# Generate title based on input type
title = self._derive_title(input_text, input_type)
# Generate filename
filename = self._generate_filename(title)
# Build target path: {context}/knowledge/inbox/{filename}
target_dir = self.monorepo_root / context / "knowledge" / "inbox"
target_dir.mkdir(parents=True, exist_ok=True)
target_path = target_dir / filename
# Build frontmatter
now = datetime.now()
frontmatter = self._build_frontmatter(
title=title,
created=now.strftime("%Y-%m-%d"),
source_type=input_type,
source_value=input_text,
tags=tags,
source_context=context,
)
# Build content body
body = self._build_body(input_text, input_type)
# Write file
content = f"{frontmatter}\n{body}\n"
target_path.write_text(content, encoding="utf-8")
return target_path
def _detect_input_type(self, input_text: str) -> str:
"""Detect if input is URL, file path, or plain text.
Args:
input_text: The raw input string.
Returns:
One of 'url', 'file', or 'text'.
"""
stripped = input_text.strip()
# Check for URL
if stripped.startswith("http://") or stripped.startswith("https://"):
return "url"
# Check for file path
if self._looks_like_filepath(stripped):
return "file"
return "text"
def _looks_like_filepath(self, text: str) -> bool:
"""Heuristic: does the text look like a file path?
Returns True if:
- It contains path separators AND has a file extension, OR
- Path(text).exists()
"""
# Contains separators and has an extension
has_separator = "/" in text or "\\" in text
has_extension = bool(re.search(r"\.\w{1,10}$", text))
if has_separator and has_extension:
return True
# Actually exists on disk
try:
return Path(text).exists()
except (OSError, ValueError):
return False
def _derive_title(self, input_text: str, input_type: str) -> str:
"""Derive a title from the input based on its type.
Args:
input_text: The raw input.
input_type: One of 'url', 'file', 'text'.
Returns:
A human-readable title string.
"""
stripped = input_text.strip()
if input_type == "url":
parsed = urlparse(stripped)
# domain + path, strip leading/trailing slashes
path_part = parsed.path.strip("/")
if path_part:
return f"{parsed.netloc}/{path_part}"
return parsed.netloc
if input_type == "file":
return Path(stripped).stem
# Text: first 60 characters
first_line = stripped.split("\n", 1)[0]
return first_line[:60]
def _generate_filename(self, title: str) -> str:
"""Generate YYYY-MM-DD-HH-MM-slugified-title.md filename.
Args:
title: The title to slugify for the filename.
Returns:
A filename string with timestamp prefix and slugified title.
"""
now = datetime.now()
timestamp = now.strftime("%Y-%m-%d-%H-%M")
slug = self._slugify(title)
# Limit slug to 50 chars
slug = slug[:50].rstrip("-")
# Fallback if slugification removed all characters (e.g. non-ASCII-only titles)
if not slug:
slug = "untitled"
return f"{timestamp}-{slug}.md"
def _slugify(self, text: str) -> str:
"""Convert text to URL-safe slug.
Lowercase, replace non-alphanumeric with hyphens,
collapse multiple hyphens, strip leading/trailing hyphens.
"""
slug = text.lower()
# Replace non-alphanumeric (keep hyphens) with hyphens
slug = re.sub(r"[^a-z0-9]+", "-", slug)
# Collapse multiple hyphens
slug = re.sub(r"-{2,}", "-", slug)
# Strip leading/trailing hyphens
slug = slug.strip("-")
return slug
def _build_frontmatter(
self,
title: str,
created: str,
source_type: str,
source_value: str,
tags: list[str],
source_context: str,
) -> str:
"""Build YAML frontmatter block.
Args:
title: Artifact title.
created: Creation date (YYYY-MM-DD).
source_type: Type of source (url, file, text).
source_value: Original input value.
tags: List of tags.
source_context: The target context.
Returns:
Complete YAML frontmatter as string including delimiters.
"""
# Escape title for YAML (wrap in quotes if it contains special chars)
escaped_title = title.replace('"', '\\"')
lines = [
"---",
"type: inbox",
f'title: "{escaped_title}"',
f"created: {created}",
"source:",
f" type: {source_type}",
f' value: "{source_value}"',
]
if tags:
tags_str = ", ".join(tags)
lines.append(f"tags: [{tags_str}]")
else:
lines.append("tags: []")
lines.append(f"source_context: {source_context}")
lines.append("---")
return "\n".join(lines)
def _build_body(self, input_text: str, input_type: str) -> str:
"""Build the Markdown content body.
Args:
input_text: The raw input.
input_type: One of 'url', 'file', 'text'.
Returns:
Markdown body content.
"""
stripped = input_text.strip()
if input_type == "url":
return f"\n[{stripped}]({stripped})\n"
if input_type == "file":
return f"\nSource file: `{stripped}`\n"
# Plain text
return f"\n{stripped}\n"
@@ -0,0 +1,355 @@
"""Config Konfigurationsmodell für die Ingestion-Pipeline.
Definiert das Schema für sources.yaml und Pipeline-Einstellungen
pro Kontext. Unterstützt Umgebungsvariablen-Auflösung (${VAR_NAME})
und Validierung der Konfiguration vor Pipeline-Start.
Requirements: 12.1, 12.2, 12.4, 12.6
"""
from __future__ import annotations
import os
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import yaml
# ---------------------------------------------------------------------------
# Exceptions
# ---------------------------------------------------------------------------
class ConfigValidationError(Exception):
"""Wird ausgelöst bei ungültiger sources.yaml-Konfiguration.
Enthält eine Liste aller gefundenen Validierungsfehler.
"""
def __init__(self, errors: list[str]) -> None:
self.errors = errors
msg = "Konfigurationsvalidierung fehlgeschlagen:\n" + "\n".join(
f" - {e}" for e in errors
)
super().__init__(msg)
# ---------------------------------------------------------------------------
# Datenmodelle
# ---------------------------------------------------------------------------
@dataclass
class SourceConfig:
"""Configuration for a single source in sources.yaml."""
type: str # confluence, jira, email, file, link, wissensdatenbank
name: str # human-readable identifier
params: dict[str, Any] = field(default_factory=dict) # type-specific params
target_folder: str = "" # override target subfolder
sync_frequency: str = "manual" # on-push, hourly, daily, manual
enabled: bool = True
@dataclass
class EnrichmentSettings:
"""Einstellungen für den Enrichment-Agenten."""
provider: str = "kiro"
model: str = ""
confidence_threshold: float = 0.7
@dataclass
class OCRSettings:
"""Einstellungen für die OCR-Engine."""
provider: str = "kiro"
languages: str = "deu+eng"
@dataclass
class GitSettings:
"""Einstellungen für die Git-Integration."""
auto_commit: bool = True
@dataclass
class PipelineSettings:
"""Pipeline-Einstellungen aus dem settings-Block in sources.yaml."""
create_tasks: bool = True
enrichment: EnrichmentSettings = field(default_factory=EnrichmentSettings)
ocr: OCRSettings = field(default_factory=OCRSettings)
git: GitSettings = field(default_factory=GitSettings)
# ---------------------------------------------------------------------------
# Umgebungsvariablen-Auflösung
# ---------------------------------------------------------------------------
_ENV_VAR_PATTERN = re.compile(r"\$\{([^}]+)\}")
def resolve_env_vars(value: str) -> str:
"""Ersetzt alle ${VAR_NAME}-Patterns in einem String durch Umgebungsvariablen.
Args:
value: Der Eingabestring mit möglichen ${VAR_NAME}-Patterns.
Returns:
Der String mit aufgelösten Umgebungsvariablen.
Raises:
ConfigValidationError: Wenn eine referenzierte Variable nicht gesetzt ist.
"""
errors: list[str] = []
def _replace(match: re.Match[str]) -> str:
var_name = match.group(1)
env_value = os.environ.get(var_name)
if env_value is None:
errors.append(
f"Umgebungsvariable '${{{var_name}}}' ist nicht gesetzt"
)
return match.group(0) # Originaltext beibehalten für Fehlermeldung
return env_value
result = _ENV_VAR_PATTERN.sub(_replace, value)
if errors:
raise ConfigValidationError(errors)
return result
def _resolve_env_vars_recursive(obj: Any) -> Any:
"""Löst ${VAR_NAME}-Patterns rekursiv in verschachtelten Datenstrukturen auf.
Traversiert dicts, Listen und Strings. Nicht-String-Werte werden
unverändert zurückgegeben.
Args:
obj: Die zu traversierende Datenstruktur.
Returns:
Die Datenstruktur mit aufgelösten Umgebungsvariablen.
Raises:
ConfigValidationError: Wenn Variablen nicht aufgelöst werden können.
"""
if isinstance(obj, str):
return resolve_env_vars(obj)
elif isinstance(obj, dict):
return {k: _resolve_env_vars_recursive(v) for k, v in obj.items()}
elif isinstance(obj, list):
return [_resolve_env_vars_recursive(item) for item in obj]
return obj
# ---------------------------------------------------------------------------
# Validierung
# ---------------------------------------------------------------------------
def _validate_source_entry(entry: dict[str, Any], index: int) -> list[str]:
"""Validiert einen einzelnen Source-Eintrag.
Args:
entry: Der rohe dict-Eintrag aus der YAML-Datei.
index: Der Index in der sources-Liste (für Fehlermeldungen).
Returns:
Liste von Validierungsfehlern (leer wenn gültig).
"""
errors: list[str] = []
if not isinstance(entry, dict):
errors.append(f"Source #{index + 1}: Eintrag muss ein Mapping sein")
return errors
# Pflichtfelder prüfen
if not entry.get("type"):
errors.append(f"Source #{index + 1}: Pflichtfeld 'type' fehlt oder ist leer")
if not entry.get("name"):
errors.append(f"Source #{index + 1}: Pflichtfeld 'name' fehlt oder ist leer")
return errors
def _collect_env_var_errors(obj: Any, path: str = "") -> list[str]:
"""Sammelt alle nicht auflösbaren Umgebungsvariablen ohne Exception.
Traversiert die Datenstruktur und identifiziert alle ${VAR_NAME}-Patterns,
deren Variablen nicht gesetzt sind.
Args:
obj: Die zu prüfende Datenstruktur.
path: Der aktuelle Pfad für Fehlermeldungen.
Returns:
Liste von Fehlermeldungen für nicht-auflösbare Variablen.
"""
errors: list[str] = []
if isinstance(obj, str):
for match in _ENV_VAR_PATTERN.finditer(obj):
var_name = match.group(1)
if os.environ.get(var_name) is None:
location = f" (in {path})" if path else ""
errors.append(
f"Umgebungsvariable '${{{var_name}}}' ist nicht gesetzt{location}"
)
elif isinstance(obj, dict):
for key, value in obj.items():
sub_path = f"{path}.{key}" if path else key
errors.extend(_collect_env_var_errors(value, sub_path))
elif isinstance(obj, list):
for i, item in enumerate(obj):
sub_path = f"{path}[{i}]"
errors.extend(_collect_env_var_errors(item, sub_path))
return errors
# ---------------------------------------------------------------------------
# sources.yaml Laden
# ---------------------------------------------------------------------------
def load_sources_config(path: Path) -> tuple[list[SourceConfig], PipelineSettings]:
"""Parst eine sources.yaml-Datei und gibt Quellkonfigurationen + Settings zurück.
Ablauf:
1. YAML-Datei einlesen
2. Pflichtfelder validieren
3. Umgebungsvariablen in allen String-Werten auflösen
4. SourceConfig- und PipelineSettings-Objekte erstellen
Args:
path: Pfad zur sources.yaml-Datei.
Returns:
Tuple aus (Liste von SourceConfig, PipelineSettings).
Raises:
FileNotFoundError: Wenn die Datei nicht existiert.
ConfigValidationError: Bei Validierungsfehlern (fehlende Felder,
nicht-auflösbare Umgebungsvariablen).
"""
if not path.exists():
raise FileNotFoundError(f"Konfigurationsdatei nicht gefunden: {path}")
text = path.read_text(encoding="utf-8")
raw: dict[str, Any] = yaml.safe_load(text) or {}
# --- Phase 1: Strukturvalidierung ---
validation_errors: list[str] = []
sources_raw = raw.get("sources", [])
if not isinstance(sources_raw, list):
validation_errors.append("'sources' muss eine Liste sein")
sources_raw = []
for i, entry in enumerate(sources_raw):
validation_errors.extend(_validate_source_entry(entry, i))
# --- Phase 2: Env-Var-Validierung (alle sammeln, nicht beim ersten abbrechen) ---
for i, entry in enumerate(sources_raw):
if isinstance(entry, dict):
env_errors = _collect_env_var_errors(
entry.get("params", {}), f"sources[{i}].params"
)
validation_errors.extend(env_errors)
# Settings-Block auch auf env vars prüfen
settings_raw = raw.get("settings", {})
if isinstance(settings_raw, dict):
env_errors = _collect_env_var_errors(settings_raw, "settings")
validation_errors.extend(env_errors)
if validation_errors:
raise ConfigValidationError(validation_errors)
# --- Phase 3: Env-Vars auflösen und Objekte erstellen ---
sources: list[SourceConfig] = []
for entry in sources_raw:
if not isinstance(entry, dict):
continue
# params mit aufgelösten env vars
resolved_params = _resolve_env_vars_recursive(entry.get("params", {}))
sources.append(
SourceConfig(
type=str(entry.get("type", "")),
name=str(entry.get("name", "")),
params=resolved_params if isinstance(resolved_params, dict) else {},
target_folder=str(entry.get("target_folder", "")),
sync_frequency=str(entry.get("sync_frequency", "manual")),
enabled=bool(entry.get("enabled", True)),
)
)
# --- Phase 4: PipelineSettings parsen ---
settings = _parse_pipeline_settings(settings_raw)
return sources, settings
def _parse_pipeline_settings(raw: Any) -> PipelineSettings:
"""Parst den settings-Block aus sources.yaml.
Args:
raw: Der rohe dict aus dem YAML.
Returns:
Ein PipelineSettings-Objekt mit Defaults für fehlende Felder.
"""
if not isinstance(raw, dict):
return PipelineSettings()
# Enrichment settings
enrichment_raw = raw.get("enrichment", {})
enrichment = EnrichmentSettings()
if isinstance(enrichment_raw, dict):
# Env vars im enrichment-Block auflösen
resolved_enrichment = _resolve_env_vars_recursive(enrichment_raw)
if isinstance(resolved_enrichment, dict):
enrichment = EnrichmentSettings(
provider=str(resolved_enrichment.get("provider", "kiro")),
model=str(resolved_enrichment.get("model", "")),
confidence_threshold=float(
resolved_enrichment.get("confidence_threshold", 0.7)
),
)
# OCR settings
ocr_raw = raw.get("ocr", {})
ocr = OCRSettings()
if isinstance(ocr_raw, dict):
resolved_ocr = _resolve_env_vars_recursive(ocr_raw)
if isinstance(resolved_ocr, dict):
ocr = OCRSettings(
provider=str(resolved_ocr.get("provider", "kiro")),
languages=str(resolved_ocr.get("languages", "deu+eng")),
)
# Git settings
git_raw = raw.get("git", {})
git = GitSettings()
if isinstance(git_raw, dict):
git = GitSettings(
auto_commit=bool(git_raw.get("auto_commit", True)),
)
return PipelineSettings(
create_tasks=bool(raw.get("create_tasks", True)),
enrichment=enrichment,
ocr=ocr,
git=git,
)
@@ -0,0 +1,692 @@
"""Enrichment KI-gestützte Anreicherung von Wissensartefakten.
Extrahiert Tags, Kategorien, Entitäten und Action-Items aus
Rohartefakten mittels LLM-Aufrufen (Kiro-first) oder regelbasierten
Heuristiken als Fallback.
Requirements: 9.1, 9.2, 9.3, 9.4, 9.5, 9.7
"""
from __future__ import annotations
import hashlib
import re
import unicodedata
from dataclasses import dataclass, field
# ---------------------------------------------------------------------------
# Data Classes
# ---------------------------------------------------------------------------
@dataclass
class ActionItem:
"""An extracted action item from content.
Attributes:
description: The action item text.
assignee: Optional person assigned to the action.
deadline: Optional ISO date string for the deadline.
hash: SHA-256 hash of description for deduplication.
"""
description: str
assignee: str | None = None
deadline: str | None = None # ISO date string
hash: str = "" # for dedup: sha256 of description
def __post_init__(self) -> None:
if not self.hash and self.description:
self.hash = _compute_action_item_hash(self.description)
@dataclass
class EnrichmentResult:
"""Output from enrichment analysis.
Attributes:
title: Extracted or generated title.
category: Content category (meeting, decision, project, reference, link, inbox).
tags: Generated tags describing the content.
people: Detected person names (confidence >= 0.7).
projects: Detected project references (confidence >= 0.7).
action_items: Extracted action items.
confidence: Overall confidence score (0.0 - 1.0).
enrichment_pending: True if LLM provider was unavailable.
"""
title: str
category: str # meeting, decision, project, reference, link, inbox
tags: list[str] = field(default_factory=list)
people: list[str] = field(default_factory=list)
projects: list[str] = field(default_factory=list)
action_items: list[ActionItem] = field(default_factory=list)
confidence: float = 0.0
enrichment_pending: bool = False
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
#: Minimum confidence threshold for including entities in output.
CONFIDENCE_THRESHOLD = 0.7
#: Valid categories for artifact classification.
VALID_CATEGORIES = ("meeting", "decision", "project", "reference", "link", "inbox")
#: Category detection keywords (German + English).
CATEGORY_KEYWORDS: dict[str, list[str]] = {
"meeting": [
"meeting", "besprechung", "jour fixe", "standup", "stand-up",
"sync", "retro", "retrospektive", "daily", "weekly",
"protokoll", "minutes", "teilnehmer", "participants",
"agenda", "termin",
],
"decision": [
"decision", "entscheidung", "beschluss", "agreed", "decided",
"resolution", "entschieden", "festgelegt", "approval",
"genehmigung", "freigabe",
],
"project": [
"projekt", "project", "milestone", "meilenstein", "sprint",
"release", "roadmap", "backlog", "epic", "feature",
"workstream", "initiative",
],
"reference": [
"referenz", "reference", "dokumentation", "documentation",
"howto", "how-to", "anleitung", "tutorial", "guide",
"handbuch", "manual", "wiki", "spec", "specification",
],
"link": [
"http://", "https://", "www.",
],
}
#: Patterns for detecting action items in text.
ACTION_ITEM_PATTERNS: list[re.Pattern[str]] = [
re.compile(r"^[-*]\s*\[\s*\]\s*(.+)", re.MULTILINE), # - [ ] task
re.compile(r"^TODO:\s*(.+)", re.MULTILINE | re.IGNORECASE), # TODO: task
re.compile(r"^ACTION:\s*(.+)", re.MULTILINE | re.IGNORECASE), # ACTION: task
re.compile(r"^→\s*(.+)", re.MULTILINE), # → task
re.compile(r"^=>\s*(.+)", re.MULTILINE), # => task
]
#: Pattern for detecting person names (Vorname Nachname).
# Supports Latin letters with diacritics (é, è, ñ, etc.).
PERSON_NAME_PATTERN = re.compile(
r"(?<![#/\[])(?:^|\s)(@?[A-ZÀ-ÖØ-Þ][a-zà-öø-ÿß]{2,}\s[A-ZÀ-ÖØ-Þ][a-zà-öø-ÿß]{2,})(?:\s|[,;.\-!?]|$)",
re.MULTILINE,
)
#: Pattern for @mentions.
AT_MENTION_PATTERN = re.compile(r"@([A-Za-zÄÖÜäöüß]+(?:\s[A-Za-zÄÖÜäöüß]+)?)")
#: Pattern for detecting project references (e.g., "Projekt XYZ", "Project: ABC").
PROJECT_REF_PATTERNS: list[re.Pattern[str]] = [
re.compile(r"(?:Projekt|Project)\s*[:\-]?\s*([A-Za-zÄÖÜäöüß0-9][\w\s\-]{2,30})", re.IGNORECASE),
re.compile(r"\[\[projects/([^\]]+)\]\]"), # existing wiki links
]
#: ISO date pattern for deadline detection.
ISO_DATE_PATTERN = re.compile(
r"(?:bis|by|until|deadline:?|fällig:?)\s*(\d{4}-\d{2}-\d{2})",
re.IGNORECASE,
)
#: Common stop words to exclude from people detection.
_STOP_NAMES: set[str] = {
"Action Items", "Quick Capture", "Knowledge Management",
"Meeting Notes", "Daily Standup", "Deutsche Bahn",
"Sprint Review", "Sprint Planning", "Open Source",
"Pull Request", "Merge Request", "Code Review",
}
# ---------------------------------------------------------------------------
# Helper Functions
# ---------------------------------------------------------------------------
def _slugify(name: str) -> str:
"""Convert a name to a URL/wiki-link slug.
Converts "André Knie""andre-knie", "Max Müller""max-mueller".
Args:
name: The name to slugify.
Returns:
Lowercase hyphenated slug suitable for wiki-links.
"""
# Normalize unicode (decompose then remove combining marks)
text = name.strip().lower()
# German umlaut replacements before NFD decomposition
replacements = {
"ä": "ae", "ö": "oe", "ü": "ue",
"ß": "ss",
}
for char, repl in replacements.items():
text = text.replace(char, repl)
# Remove remaining diacritics
text = unicodedata.normalize("NFD", text)
text = "".join(c for c in text if unicodedata.category(c) != "Mn")
# Replace non-alphanumeric with hyphens
text = re.sub(r"[^a-z0-9]+", "-", text)
# Collapse multiple hyphens and strip leading/trailing
text = re.sub(r"-+", "-", text).strip("-")
return text
def _compute_action_item_hash(description: str) -> str:
"""Compute SHA-256 hash of an action item description for deduplication.
Args:
description: The action item description text.
Returns:
First 16 characters of hex SHA-256 digest.
"""
normalized = description.strip().lower()
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
def _detect_category(content: str, source_type: str = "") -> tuple[str, float]:
"""Detect the content category using keyword matching.
Args:
content: The text content to classify.
source_type: Optional source type hint (e.g., "email", "confluence").
Returns:
Tuple of (category, confidence).
"""
content_lower = content.lower()
# Source type can give strong hints
source_type_hints: dict[str, str] = {
"email": "reference",
"chat": "reference",
"confluence": "reference",
"jira": "project",
}
scores: dict[str, float] = {cat: 0.0 for cat in VALID_CATEGORIES}
# Count keyword matches per category
for category, keywords in CATEGORY_KEYWORDS.items():
for keyword in keywords:
if keyword in content_lower:
scores[category] += 1.0
# Apply source type hint bonus
if source_type and source_type.lower() in source_type_hints:
hinted = source_type_hints[source_type.lower()]
scores[hinted] += 0.5
# Check if content is primarily a URL/link
stripped = content.strip()
if stripped.startswith(("http://", "https://")) and "\n" not in stripped:
return "link", 0.9
# Find best category
best_category = max(scores, key=lambda k: scores[k])
best_score = scores[best_category]
if best_score == 0.0:
return "inbox", 0.3
# Normalize confidence (more matches = higher confidence, capped at 0.95)
confidence = min(0.95, 0.5 + best_score * 0.1)
return best_category, confidence
def _extract_people(content: str) -> list[tuple[str, float]]:
"""Extract person names from content with confidence scores.
Detects patterns like "Max Müller", "@andre.knie", "@André Knie".
Args:
content: The text to analyze.
Returns:
List of (name, confidence) tuples.
"""
people: dict[str, float] = {}
# Detect @mentions (high confidence)
for match in AT_MENTION_PATTERN.finditer(content):
name = match.group(1).strip()
if len(name) >= 3 and name not in _STOP_NAMES:
people[name] = max(people.get(name, 0), 0.9)
# Detect "Vorname Nachname" patterns
for match in PERSON_NAME_PATTERN.finditer(content):
name = match.group(1).strip().lstrip("@")
if name in _STOP_NAMES:
continue
if len(name) >= 5: # at least "Xx Yy" length
# Lower confidence for bare name patterns (could be titles, places)
people[name] = max(people.get(name, 0), 0.75)
return [(name, conf) for name, conf in people.items()]
def _extract_projects(content: str) -> list[tuple[str, float]]:
"""Extract project references from content with confidence scores.
Args:
content: The text to analyze.
Returns:
List of (project_name, confidence) tuples.
"""
projects: dict[str, float] = {}
for pattern in PROJECT_REF_PATTERNS:
for match in pattern.finditer(content):
proj = match.group(1).strip().rstrip(".,;:!?")
if len(proj) >= 2:
projects[proj] = max(projects.get(proj, 0), 0.8)
return [(name, conf) for name, conf in projects.items()]
def _extract_action_items(content: str) -> list[ActionItem]:
"""Extract action items from content.
Detects patterns like:
- [ ] Do something
TODO: Something else
ACTION: Assign this task
→ Next step
Args:
content: The text to analyze.
Returns:
List of ActionItem instances.
"""
seen_hashes: set[str] = set()
items: list[ActionItem] = []
for pattern in ACTION_ITEM_PATTERNS:
for match in pattern.finditer(content):
description = match.group(1).strip()
if not description or len(description) < 3:
continue
item_hash = _compute_action_item_hash(description)
if item_hash in seen_hashes:
continue
seen_hashes.add(item_hash)
# Try to extract assignee from description
assignee = _extract_assignee(description)
# Try to extract deadline
deadline = _extract_deadline(description, content)
items.append(ActionItem(
description=description,
assignee=assignee,
deadline=deadline,
hash=item_hash,
))
return items
def _extract_assignee(description: str) -> str | None:
"""Try to extract an assignee from action item text.
Looks for patterns like "@Name" or "(Name)" at the end.
Args:
description: The action item description.
Returns:
Assignee name or None.
"""
# Check for @mention in description (supports diacritics)
at_match = re.search(r"@([\w\u00C0-\u024F]+(?:\s[\w\u00C0-\u024F]+)?)", description)
if at_match:
return at_match.group(1)
# Check for (Name) at end
paren_match = re.search(
r"\(([A-ZÀ-ÖØ-Þ][\w\u00C0-\u024F]+(?:\s[A-ZÀ-ÖØ-Þ][\w\u00C0-\u024F]+)?)\)\s*$",
description,
)
if paren_match:
return paren_match.group(1)
return None
def _extract_deadline(description: str, full_content: str = "") -> str | None:
"""Try to extract a deadline from action item or surrounding context.
Args:
description: The action item description.
full_content: The full document content for context.
Returns:
ISO date string or None.
"""
# Look in description first
date_match = ISO_DATE_PATTERN.search(description)
if date_match:
return date_match.group(1)
# Simple date pattern in description
simple_date = re.search(r"(\d{4}-\d{2}-\d{2})", description)
if simple_date:
return simple_date.group(1)
return None
def _extract_title(content: str) -> str:
"""Extract title from first heading or first line of content.
Args:
content: The text content.
Returns:
Extracted title string.
"""
lines = content.strip().split("\n")
for line in lines[:5]: # Check first 5 lines
line = line.strip()
if not line:
continue
# Markdown heading
heading_match = re.match(r"^#{1,3}\s+(.+)", line)
if heading_match:
return heading_match.group(1).strip()
# First non-empty line as fallback title
if len(line) > 2:
# Truncate very long first lines
return line[:80].rstrip(".,;:!?")
return "Untitled"
def _extract_tags(content: str, category: str) -> list[str]:
"""Extract relevant tags from content using keyword frequency.
Args:
content: The text content to analyze.
category: The detected category (used as base tag).
Returns:
List of tag strings.
"""
tags: list[str] = []
# Add category as a tag if meaningful
if category and category != "inbox":
tags.append(category)
content_lower = content.lower()
# Domain-specific tag detection
tech_keywords = {
"python", "java", "typescript", "docker", "kubernetes", "k8s",
"api", "rest", "graphql", "ci/cd", "pipeline", "terraform",
"aws", "azure", "cloud", "microservice", "frontend", "backend",
"database", "sql", "mongodb", "redis",
}
bahn_keywords = {
"infrago", "db", "confluence", "jira", "openshift",
"pipeship", "cnp", "einfachbahn",
}
for kw in tech_keywords:
if kw in content_lower:
tags.append(kw)
for kw in bahn_keywords:
if kw in content_lower:
tags.append(kw)
# Limit tags to reasonable number
return tags[:10]
def generate_wiki_links(people: list[str], projects: list[str]) -> dict[str, list[str]]:
"""Generate wiki-links for detected entities.
Args:
people: List of detected person names.
projects: List of detected project names.
Returns:
Dict with 'people' and 'projects' wiki-link lists.
"""
people_links = [f"[[people/{_slugify(name)}]]" for name in people]
project_links = [f"[[projects/{_slugify(proj)}]]" for proj in projects]
return {"people": people_links, "projects": project_links}
def generate_action_items_section(action_items: list[ActionItem]) -> str:
"""Generate a markdown section for action items.
Args:
action_items: List of ActionItem instances.
Returns:
Markdown string with ## Action Items heading and list.
"""
if not action_items:
return ""
lines = ["## Action Items", ""]
for item in action_items:
parts = [f"- [ ] {item.description}"]
annotations: list[str] = []
if item.assignee:
annotations.append(f"@{item.assignee}")
if item.deadline:
annotations.append(f"bis {item.deadline}")
if annotations:
parts[0] += f" ({', '.join(annotations)})"
lines.append(parts[0])
lines.append("") # trailing newline
return "\n".join(lines)
# ---------------------------------------------------------------------------
# EnrichmentAgent
# ---------------------------------------------------------------------------
class EnrichmentAgent:
"""Content analysis and enrichment agent.
Provider-Strategie (Priorität):
1. Kiro AI-Agent (primär) nutzt den laufenden Orchestrator-Agenten direkt,
keine externe Abhängigkeit, kein API-Key nötig
2. Regelbasierte Heuristiken als Fallback wenn kein LLM verfügbar
Im aktuellen MVP werden nur die regelbasierten Heuristiken verwendet.
Die LLM-Integration (Kiro/LiteLLM) wird als Stub vorgehalten.
"""
def __init__(self, provider: str = "kiro", model: str = "") -> None:
"""Initialize with LLM config.
Args:
provider: LLM provider ('kiro' or 'litellm'). Defaults to 'kiro'.
model: Model identifier (empty = provider default).
"""
self.provider = provider
self.model = model
self._provider_available = self._check_provider_available()
def _check_provider_available(self) -> bool:
"""Check if the configured LLM provider is available.
For now, rule-based heuristics are always available.
The Kiro/LiteLLM check is a stub for future implementation.
Returns:
True if provider is available, False otherwise.
"""
# Rule-based heuristics are always available
# When LLM providers are implemented, this will actually check connectivity
if self.provider == "kiro":
# Kiro provider: in MVP we fall back to heuristics which are always available
return True
elif self.provider == "litellm":
# LiteLLM would require env vars LLM_PROVIDER, LLM_MODEL
# Stub: not available in MVP
return False
return False
def enrich(self, content: str, source_type: str = "") -> EnrichmentResult:
"""Analyze content and generate metadata.
Uses rule-based heuristics to extract:
- Title from first heading or first line
- Category from keyword matching
- Tags from domain keywords
- People from name patterns and @mentions
- Projects from explicit references
- Action items from TODO/checkbox patterns
Only entities with confidence >= 0.7 are included in the output.
Args:
content: The text content to analyze.
source_type: Optional hint about the source type.
Returns:
EnrichmentResult with extracted metadata.
"""
if not content or not content.strip():
return EnrichmentResult(
title="Untitled",
category="inbox",
confidence=0.0,
enrichment_pending=not self._provider_available,
)
# If provider is not available, return minimal result with pending flag
if not self._provider_available:
title = _extract_title(content)
return EnrichmentResult(
title=title,
category="inbox",
confidence=0.0,
enrichment_pending=True,
)
# Extract title
title = _extract_title(content)
# Detect category
category, cat_confidence = _detect_category(content, source_type)
# Extract tags
tags = _extract_tags(content, category)
# Extract people (apply confidence threshold)
raw_people = _extract_people(content)
people = [name for name, conf in raw_people if conf >= CONFIDENCE_THRESHOLD]
# Extract projects (apply confidence threshold)
raw_projects = _extract_projects(content)
projects = [name for name, conf in raw_projects if conf >= CONFIDENCE_THRESHOLD]
# Extract action items
action_items = _extract_action_items(content)
# Compute overall confidence
# Based on: how much structure was found in the content
signals = [
cat_confidence,
min(1.0, len(tags) * 0.2) if tags else 0.0,
0.8 if people else 0.0,
0.8 if action_items else 0.0,
]
overall_confidence = sum(signals) / len(signals) if signals else 0.3
return EnrichmentResult(
title=title,
category=category,
tags=tags,
people=people,
projects=projects,
action_items=action_items,
confidence=round(overall_confidence, 2),
enrichment_pending=False,
)
def classify_relevance(self, content: str) -> float:
"""Score content relevance (0.0 = irrelevant, 1.0 = highly relevant).
Simple heuristic based on:
- Content length (very short = likely low relevance)
- Structural elements (headings, lists, links)
- Domain keywords presence
- Action items presence
Args:
content: The text content to score.
Returns:
Float between 0.0 and 1.0 indicating relevance.
"""
if not content or not content.strip():
return 0.0
score = 0.0
content_stripped = content.strip()
# Length-based scoring
length = len(content_stripped)
if length < 20:
score += 0.1
elif length < 100:
score += 0.3
elif length < 500:
score += 0.5
elif length < 2000:
score += 0.7
else:
score += 0.8
# Structural elements bonus
has_headings = bool(re.search(r"^#+\s", content, re.MULTILINE))
has_lists = bool(re.search(r"^[-*]\s", content, re.MULTILINE))
has_links = bool(re.search(r"\[.+?\]\(.+?\)", content))
if has_headings:
score += 0.1
if has_lists:
score += 0.05
if has_links:
score += 0.05
# Action items bonus
for pattern in ACTION_ITEM_PATTERNS:
if pattern.search(content):
score += 0.1
break
# Cap at 1.0
return min(1.0, round(score, 2))
@@ -0,0 +1,253 @@
"""Git-Integration Automatisches Committen von Wissensänderungen.
Erstellt strukturierte Git-Commits nach erfolgreicher Ingestion
mit konventionellen Commit-Messages.
Commit-Message-Format:
knowledge({context}): {action} {count} artifacts from {source}
knowledge({context}): capture "{title}"
Verwendet subprocess für Git-Aufrufe (kein gitpython).
Fehler werden graceful behandelt: Warnung loggen, Dateien beibehalten.
Requirements: 16.1, 16.2, 16.3, 16.4, 16.5
"""
from __future__ import annotations
import logging
import subprocess
from pathlib import Path
logger = logging.getLogger(__name__)
def generate_commit_message(
context: str,
action: str,
count: int,
source: str,
) -> str:
"""Generiert eine konventionelle Commit-Message für Knowledge-Änderungen.
Format: knowledge({context}): {action} {count} artifacts from {source}
Args:
context: Der Arbeitskontext (bahn, dhive, privat).
action: Die durchgeführte Aktion (ingest, capture, update, etc.).
count: Anzahl der verarbeiteten Artefakte.
source: Die Quelle der Artefakte (confluence, jira, inbox, etc.).
Returns:
Die formatierte Commit-Message.
"""
return f"knowledge({context}): {action} {count} artifacts from {source}"
def generate_capture_commit_message(context: str, title: str) -> str:
"""Generiert eine Commit-Message für Quick-Capture-Artefakte.
Format: knowledge({context}): capture "{title}"
Args:
context: Der Arbeitskontext (bahn, dhive, privat).
title: Der Titel des erfassten Artefakts.
Returns:
Die formatierte Commit-Message.
"""
return f'knowledge({context}): capture "{title}"'
def auto_commit(
context: str,
action: str,
count: int,
source: str,
paths: list[Path],
*,
no_commit: bool = False,
auto_commit_enabled: bool = True,
cwd: Path | None = None,
) -> str:
"""Staged und committed Änderungen an den angegebenen Pfaden.
Respektiert --no-commit-Flag und git.auto_commit-Config.
Behandelt Git-Fehler graceful (Warnung, Dateien bleiben erhalten).
Args:
context: Der Arbeitskontext (bahn, dhive, privat).
action: Die durchgeführte Aktion (ingest, capture, etc.).
count: Anzahl der verarbeiteten Artefakte.
source: Die Quelle der Artefakte.
paths: Liste der zu committenden Dateipfade.
no_commit: Wenn True, wird kein Commit erzeugt (--no-commit Flag).
auto_commit_enabled: Wert aus git.auto_commit in sources.yaml.
cwd: Arbeitsverzeichnis für Git-Befehle. Wenn None, wird das
übergeordnete Verzeichnis des ersten Pfades verwendet.
Returns:
Den Commit-SHA bei Erfolg, oder einen leeren String wenn kein
Commit erzeugt wurde (no_commit, disabled, Fehler, keine Pfade).
"""
# Keine Pfade → nichts zu committen
if not paths:
logger.debug("auto_commit: Keine Pfade angegeben, überspringe Commit.")
return ""
# --no-commit Flag hat höchste Priorität
if no_commit:
logger.info("auto_commit: --no-commit gesetzt, überspringe Git-Commit.")
return ""
# git.auto_commit aus Config prüfen
if not auto_commit_enabled:
logger.info(
"auto_commit: git.auto_commit ist deaktiviert, überspringe Git-Commit."
)
return ""
# Arbeitsverzeichnis bestimmen
if cwd is None:
cwd = paths[0].parent
# Commit-Message generieren
message = generate_commit_message(context, action, count, source)
return _execute_git_commit(paths, message, cwd)
def auto_commit_capture(
context: str,
title: str,
paths: list[Path],
*,
no_commit: bool = False,
auto_commit_enabled: bool = True,
cwd: Path | None = None,
) -> str:
"""Staged und committed ein Quick-Capture-Artefakt.
Args:
context: Der Arbeitskontext (bahn, dhive, privat).
title: Der Titel des erfassten Artefakts.
paths: Liste der zu committenden Dateipfade.
no_commit: Wenn True, wird kein Commit erzeugt.
auto_commit_enabled: Wert aus git.auto_commit in sources.yaml.
cwd: Arbeitsverzeichnis für Git-Befehle.
Returns:
Den Commit-SHA bei Erfolg, oder einen leeren String.
"""
if not paths:
return ""
if no_commit:
logger.info("auto_commit_capture: --no-commit gesetzt, überspringe.")
return ""
if not auto_commit_enabled:
logger.info("auto_commit_capture: git.auto_commit deaktiviert, überspringe.")
return ""
if cwd is None:
cwd = paths[0].parent
message = generate_capture_commit_message(context, title)
return _execute_git_commit(paths, message, cwd)
def _execute_git_commit(
paths: list[Path],
message: str,
cwd: Path,
) -> str:
"""Führt git add + git commit aus.
Bei Fehlern wird eine Warnung geloggt und ein leerer String zurückgegeben.
Die erstellten Dateien bleiben in jedem Fall erhalten.
Args:
paths: Zu stagenden Dateipfade.
message: Die Commit-Message.
cwd: Arbeitsverzeichnis für Git-Befehle.
Returns:
Den Commit-SHA bei Erfolg, oder einen leeren String bei Fehler.
"""
try:
# Stage files
str_paths = [str(p) for p in paths]
_run_git(["add", "--"] + str_paths, cwd=cwd)
# Commit
_run_git(["commit", "-m", message], cwd=cwd)
# SHA des letzten Commits holen
result = _run_git(["rev-parse", "HEAD"], cwd=cwd)
sha = result.stdout.strip()
logger.info(f"Git-Commit erstellt: {sha[:8]} {message}")
return sha
except GitCommandError as e:
logger.warning(
f"Git-Commit fehlgeschlagen: {e}. "
"Dateien wurden erstellt, aber nicht committed."
)
return ""
def _run_git(
args: list[str],
cwd: Path,
) -> subprocess.CompletedProcess[str]:
"""Führt einen Git-Befehl als Subprocess aus.
Args:
args: Git-Subcommand und Argumente (ohne 'git' Präfix).
cwd: Arbeitsverzeichnis.
Returns:
Das CompletedProcess-Objekt bei Erfolg.
Raises:
GitCommandError: Wenn der Git-Befehl fehlschlägt.
"""
cmd = ["git"] + args
try:
result = subprocess.run(
cmd,
cwd=str(cwd),
capture_output=True,
text=True,
timeout=30,
)
except FileNotFoundError:
raise GitCommandError(
"Git ist nicht installiert oder nicht im PATH verfügbar."
)
except subprocess.TimeoutExpired:
raise GitCommandError(f"Git-Befehl hat Timeout überschritten: {cmd}")
except OSError as e:
raise GitCommandError(f"OS-Fehler beim Ausführen von Git: {e}")
if result.returncode != 0:
stderr = result.stderr.strip()
raise GitCommandError(
f"Git-Befehl fehlgeschlagen (exit {result.returncode}): "
f"{' '.join(cmd)}\n{stderr}"
)
return result
class GitCommandError(Exception):
"""Wird bei fehlgeschlagenen Git-Befehlen ausgelöst.
Diese Exception wird intern in _execute_git_commit abgefangen
und in eine Warnung umgewandelt.
"""
pass
@@ -0,0 +1,715 @@
"""IngestionPipeline Orchestrator für die Wissensaufnahme.
Koordiniert den gesamten Ingestion-Workflow: Quellen laden, extrahieren,
anreichern, speichern und indexieren.
Requirements: 2.2, 2.3, 2.4, 2.5, 2.6, 3.4, 3.5, 3.6, 12.3
"""
from __future__ import annotations
import hashlib
import logging
from dataclasses import dataclass, field
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any
import yaml
from monorepo.knowledge.artifact import (
Artifact,
ArtifactMetadata,
compute_content_hash,
write_artifact,
)
from monorepo.knowledge.index import IndexEntry, YAMLIndex
from monorepo.knowledge.ingestion.config import (
PipelineSettings,
SourceConfig,
load_sources_config,
)
from monorepo.knowledge.ingestion.enrichment import (
ActionItem,
EnrichmentAgent,
EnrichmentResult,
)
from monorepo.knowledge.ingestion.git_integration import auto_commit
from monorepo.knowledge.ingestion.router import ContextRouter, category_to_folder
from monorepo.knowledge.integrations.orgmylife import OrgMyLifeIntegration
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceError,
SourceStrategy,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Data Models
# ---------------------------------------------------------------------------
@dataclass
class IngestError:
"""Structured error from ingestion pipeline."""
source: str
error_type: str
message: str
timestamp: str = ""
retry: bool = False
artifact_path: str = ""
def __post_init__(self) -> None:
if not self.timestamp:
self.timestamp = datetime.now(timezone.utc).isoformat()
@dataclass
class IngestionResult:
"""Result of a full ingestion run."""
context: str
processed: int = 0
updated: int = 0
skipped: int = 0
errors: list[IngestError] = field(default_factory=list)
tasks_created: int = 0
commit_sha: str = ""
# ---------------------------------------------------------------------------
# Source Registry
# ---------------------------------------------------------------------------
# Maps source type names to SourceStrategy classes.
# Populated by register_source_strategy() or by the pipeline at startup.
_SOURCE_REGISTRY: dict[str, type[SourceStrategy]] = {}
def register_source_strategy(source_type: str, strategy_class: type[SourceStrategy]) -> None:
"""Register a source strategy class for a given source type.
Args:
source_type: The source type name (e.g., 'markdown', 'confluence').
strategy_class: The SourceStrategy subclass to use for this type.
"""
_SOURCE_REGISTRY[source_type] = strategy_class
def get_source_strategy(source_type: str) -> SourceStrategy | None:
"""Get an instantiated source strategy for the given type.
Args:
source_type: The source type name.
Returns:
An instance of the registered strategy, or None if not registered.
"""
strategy_class = _SOURCE_REGISTRY.get(source_type)
if strategy_class is None:
return None
return strategy_class()
def register_default_strategies() -> None:
"""Register all built-in source strategies.
This is called automatically when an IngestionPipeline is created,
ensuring all standard source types are available.
"""
from monorepo.knowledge.sources.chat import ChatSource
from monorepo.knowledge.sources.confluence import ConfluenceSource
from monorepo.knowledge.sources.email import EmailSource
from monorepo.knowledge.sources.jira import JiraSource
from monorepo.knowledge.sources.markdown import MarkdownSource
from monorepo.knowledge.sources.pdf import PDFSource
from monorepo.knowledge.sources.wissensdatenbank import WissensdatenbankSource
defaults: dict[str, type[SourceStrategy]] = {
"markdown": MarkdownSource,
"pdf": PDFSource,
"confluence": ConfluenceSource,
"jira": JiraSource,
"email": EmailSource,
"chat": ChatSource,
"wissensdatenbank": WissensdatenbankSource,
}
for source_type, strategy_class in defaults.items():
if source_type not in _SOURCE_REGISTRY:
_SOURCE_REGISTRY[source_type] = strategy_class
# ---------------------------------------------------------------------------
# IngestionPipeline
# ---------------------------------------------------------------------------
class IngestionPipeline:
"""Orchestrates the full ingestion workflow.
Pipeline steps:
1. Load sources.yaml configuration
2. For each enabled source: extract artifacts via SourceStrategy
3. Enrich artifacts (titles, categories, tags, entities)
4. Store artifacts in the correct knowledge folder
5. Update YAML index
6. Optionally commit changes via git
Fail-forward: individual source failures are logged and skipped,
remaining sources continue processing.
"""
def __init__(
self,
monorepo_root: Path,
context: str,
dry_run: bool = False,
no_commit: bool = False,
enrichment_agent: EnrichmentAgent | None = None,
source_strategies: dict[str, SourceStrategy] | None = None,
) -> None:
"""Initialize the ingestion pipeline.
Args:
monorepo_root: Root path of the monorepo.
context: Target context (bahn, dhive, privat).
dry_run: If True, do not write files.
no_commit: If True, skip git commit.
enrichment_agent: Optional custom enrichment agent (for testing).
source_strategies: Optional custom strategy mapping (for testing).
"""
self.monorepo_root = monorepo_root
self.context = context
self.dry_run = dry_run
self.no_commit = no_commit
self.router = ContextRouter(monorepo_root)
self.enrichment_agent = enrichment_agent or EnrichmentAgent()
self._source_strategies = source_strategies or {}
# Ensure all built-in source strategies are registered
register_default_strategies()
@property
def knowledge_path(self) -> Path:
"""Path to the context's knowledge folder."""
return self.monorepo_root / self.context / "knowledge"
@property
def index_path(self) -> Path:
"""Path to the context's YAML index."""
return self.knowledge_path / "_index.yaml"
@property
def sources_config_path(self) -> Path:
"""Path to the context's sources.yaml."""
return self.knowledge_path / "sources.yaml"
def _get_strategy(self, source_type: str) -> SourceStrategy | None:
"""Get a source strategy, checking local overrides first.
Args:
source_type: The source type name.
Returns:
A SourceStrategy instance or None.
"""
if source_type in self._source_strategies:
return self._source_strategies[source_type]
return get_source_strategy(source_type)
def _load_index(self) -> YAMLIndex:
"""Load the YAML index for this context."""
index = YAMLIndex(self.index_path)
index.load()
return index
def _is_unchanged(self, artifact: Artifact, index: YAMLIndex) -> bool:
"""Check if an artifact is unchanged based on content hash.
Args:
artifact: The artifact to check.
index: The current YAML index.
Returns:
True if the artifact's content hash matches the index entry.
"""
content_hash = compute_content_hash(artifact.content)
artifact_id = self._compute_artifact_id(artifact)
existing = index.get_entry(artifact_id)
if existing and existing.content_hash == content_hash:
return True
return False
def _compute_artifact_id(self, artifact: Artifact) -> str:
"""Compute a unique ID for an artifact.
Format: {context}/{type}/{filename_stem}
Args:
artifact: The artifact.
Returns:
String ID for the artifact.
"""
stem = artifact.file_path.stem if artifact.file_path.stem else "untitled"
art_type = artifact.metadata.type or artifact.metadata.category or "inbox"
return f"{self.context}/{art_type}/{stem}"
def _enrich_artifact(self, artifact: Artifact) -> Artifact:
"""Enrich an artifact with metadata from the enrichment agent.
Args:
artifact: The raw artifact.
Returns:
The artifact with enriched metadata.
"""
result = self.enrichment_agent.enrich(
artifact.content, artifact.metadata.source.get("type", "")
)
# Apply enrichment results to metadata
if not artifact.metadata.title or artifact.metadata.title == "Untitled":
artifact.metadata.title = result.title
if not artifact.metadata.category:
artifact.metadata.category = result.category
if not artifact.metadata.tags:
artifact.metadata.tags = result.tags
if not artifact.metadata.people:
artifact.metadata.people = [
f"[[people/{_slugify_simple(p)}]]" for p in result.people
]
if not artifact.metadata.projects:
artifact.metadata.projects = [
f"[[projects/{_slugify_simple(p)}]]" for p in result.projects
]
artifact.metadata.enrichment_pending = result.enrichment_pending
# Compute content hash
artifact.metadata.content_hash = compute_content_hash(artifact.content)
# Set source context
if not artifact.metadata.source_context:
artifact.metadata.source_context = self.context
return artifact
def _store_artifact(self, artifact: Artifact) -> Path:
"""Store an artifact in the knowledge folder.
Args:
artifact: The enriched artifact.
Returns:
The path where the artifact was stored.
"""
category = artifact.metadata.category or "inbox"
folder = category_to_folder(category)
target_dir = self.knowledge_path / folder
if not self.dry_run:
target_dir.mkdir(parents=True, exist_ok=True)
# Determine filename
filename = artifact.file_path.name
if not filename:
filename = "untitled.md"
target_path = target_dir / filename
if not self.dry_run:
# Write the artifact file
from monorepo.knowledge.artifact import _generate_frontmatter
frontmatter = _generate_frontmatter(artifact.metadata)
content = frontmatter + artifact.content
target_path.write_text(content, encoding="utf-8")
artifact.file_path = target_path
return target_path
def _update_index(self, artifact: Artifact, index: YAMLIndex) -> None:
"""Update the YAML index with the artifact's metadata.
Args:
artifact: The processed artifact.
index: The YAML index to update.
"""
artifact_id = self._compute_artifact_id(artifact)
entry = IndexEntry(
id=artifact_id,
title=artifact.metadata.title,
type=artifact.metadata.type or artifact.metadata.category or "inbox",
tags=artifact.metadata.tags,
scope=self.context,
summary="",
path=str(artifact.file_path),
content_hash=artifact.metadata.content_hash,
links=[link.target for link in artifact.metadata.links],
)
index.update_entry(entry)
def run(self) -> IngestionResult:
"""Execute full pipeline: load config → extract → enrich → store → index → commit → tasks.
Returns:
IngestionResult with processing statistics.
"""
result = IngestionResult(context=self.context)
# Load index
index = self._load_index()
# Load source configurations
sources: list[SourceConfig] = []
settings = PipelineSettings()
if self.sources_config_path.exists():
try:
sources, settings = load_sources_config(self.sources_config_path)
except Exception as e:
result.errors.append(IngestError(
source="sources.yaml",
error_type="config",
message=str(e),
))
return result
# Track written paths for git commit
written_paths: list[Path] = []
# Track action items for OrgMyLife task creation
all_action_items: list[tuple[str, list[ActionItem], str]] = []
# Process each enabled source
for source_config in sources:
if not source_config.enabled:
continue
strategy = self._get_strategy(source_config.type)
if strategy is None:
logger.warning(
f"No strategy registered for source type '{source_config.type}'"
)
result.errors.append(IngestError(
source=source_config.name,
error_type="config",
message=f"No strategy for type '{source_config.type}'",
))
continue
# Extract artifacts from source
try:
extraction = strategy.extract(source_config, self.context)
except Exception as e:
logger.error(
f"Source '{source_config.name}' failed: {e}"
)
result.errors.append(IngestError(
source=source_config.name,
error_type="extraction",
message=str(e),
retry=True,
))
continue
# Record extraction errors
for src_error in extraction.errors:
result.errors.append(IngestError(
source=src_error.source_name,
error_type=src_error.error_type,
message=src_error.message,
timestamp=src_error.timestamp,
retry=src_error.retry,
))
result.skipped += extraction.skipped
# Process each extracted artifact
for artifact in extraction.artifacts:
# Check if unchanged (idempotent ingestion)
if self._is_unchanged(artifact, index):
result.skipped += 1
continue
# Enrich
artifact = self._enrich_artifact(artifact)
# Collect action items for OrgMyLife task creation
enrichment_result = self.enrichment_agent.enrich(
artifact.content, artifact.metadata.source.get("type", "")
)
if enrichment_result.action_items:
artifact_id = self._compute_artifact_id(artifact)
all_action_items.append((
artifact_id,
enrichment_result.action_items,
str(artifact.file_path),
))
# Store
if not self.dry_run:
stored_path = self._store_artifact(artifact)
written_paths.append(stored_path)
# Update index
self._update_index(artifact, index)
result.processed += 1
# Save index
if not self.dry_run:
index.save()
written_paths.append(self.index_path)
# Git auto-commit (Requirement 16.1)
if not self.dry_run and written_paths:
commit_sha = auto_commit(
context=self.context,
action="ingest",
count=result.processed,
source="pipeline",
paths=written_paths,
no_commit=self.no_commit,
auto_commit_enabled=settings.git.auto_commit,
cwd=self.monorepo_root,
)
result.commit_sha = commit_sha
# OrgMyLife task creation for action items (Requirement 10.1)
if not self.dry_run and settings.create_tasks and all_action_items:
try:
orgmylife = OrgMyLifeIntegration(
context_path=self.knowledge_path,
enabled=settings.create_tasks,
)
for artifact_id, action_items, source_path in all_action_items:
mappings = orgmylife.create_tasks(
artifact_id=artifact_id,
action_items=action_items,
source_path=source_path,
)
result.tasks_created += len(mappings)
except Exception as e:
logger.warning(f"OrgMyLife task creation failed: {e}")
result.errors.append(IngestError(
source="orgmylife",
error_type="integration",
message=str(e),
))
return result
def run_with_artifacts(
self,
artifacts: list[Artifact],
source_configs: list[SourceConfig] | None = None,
) -> IngestionResult:
"""Run pipeline with pre-extracted artifacts (for testing/programmatic use).
Args:
artifacts: Pre-extracted artifacts to process.
source_configs: Optional source configs (not used for extraction).
Returns:
IngestionResult with processing statistics.
"""
result = IngestionResult(context=self.context)
index = self._load_index()
for artifact in artifacts:
# Check if unchanged (idempotent ingestion)
if self._is_unchanged(artifact, index):
result.skipped += 1
continue
# Enrich
artifact = self._enrich_artifact(artifact)
# Store
if not self.dry_run:
self._store_artifact(artifact)
# Update index
self._update_index(artifact, index)
result.processed += 1
# Save index
if not self.dry_run:
index.save()
return result
def process_inbox(self) -> IngestionResult:
"""Process all items in {context}/knowledge/inbox/.
Reads all markdown files from the inbox, enriches them,
and moves them to the appropriate category folder.
After processing, commits changes and creates tasks for action items.
Returns:
IngestionResult with processing statistics.
"""
result = IngestionResult(context=self.context)
inbox_path = self.knowledge_path / "inbox"
if not inbox_path.exists():
return result
index = self._load_index()
# Load settings for git/task config
settings = PipelineSettings()
if self.sources_config_path.exists():
try:
_, settings = load_sources_config(self.sources_config_path)
except Exception:
pass # Use defaults if config load fails
# Track written paths for git commit
written_paths: list[Path] = []
# Track action items for OrgMyLife
all_action_items: list[tuple[str, list[ActionItem], str]] = []
for file_path in inbox_path.glob("*.md"):
if file_path.name == "index.md":
continue
try:
content = file_path.read_text(encoding="utf-8")
except Exception as e:
result.errors.append(IngestError(
source=str(file_path),
error_type="read",
message=str(e),
))
continue
# Create artifact from inbox file
artifact = Artifact(
metadata=ArtifactMetadata(
type="inbox",
title=file_path.stem,
source_context=self.context,
created=date.today(),
),
content=content,
file_path=file_path,
)
# Check if unchanged
if self._is_unchanged(artifact, index):
result.skipped += 1
continue
# Enrich
artifact = self._enrich_artifact(artifact)
# Collect action items for OrgMyLife
enrichment_result = self.enrichment_agent.enrich(
artifact.content, "inbox"
)
if enrichment_result.action_items:
artifact_id = self._compute_artifact_id(artifact)
all_action_items.append((
artifact_id,
enrichment_result.action_items,
str(artifact.file_path),
))
# Store in categorized folder
if not self.dry_run:
stored_path = self._store_artifact(artifact)
written_paths.append(stored_path)
# Update index
self._update_index(artifact, index)
result.processed += 1
# Save index
if not self.dry_run:
index.save()
if result.processed > 0:
written_paths.append(self.index_path)
# Git auto-commit
if not self.dry_run and written_paths:
commit_sha = auto_commit(
context=self.context,
action="categorize",
count=result.processed,
source="inbox",
paths=written_paths,
no_commit=self.no_commit,
auto_commit_enabled=settings.git.auto_commit,
cwd=self.monorepo_root,
)
result.commit_sha = commit_sha
# OrgMyLife task creation for action items
if not self.dry_run and settings.create_tasks and all_action_items:
try:
orgmylife = OrgMyLifeIntegration(
context_path=self.knowledge_path,
enabled=settings.create_tasks,
)
for artifact_id, action_items, source_path in all_action_items:
mappings = orgmylife.create_tasks(
artifact_id=artifact_id,
action_items=action_items,
source_path=source_path,
)
result.tasks_created += len(mappings)
except Exception as e:
logger.warning(f"OrgMyLife task creation failed: {e}")
result.errors.append(IngestError(
source="orgmylife",
error_type="integration",
message=str(e),
))
return result
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _slugify_simple(name: str) -> str:
"""Simple slugification for wiki-links.
Args:
name: The name to slugify.
Returns:
Lowercase, hyphen-separated string.
"""
import re
import unicodedata
text = name.strip().lower()
# German umlaut replacements
replacements = {"ä": "ae", "ö": "oe", "ü": "ue", "ß": "ss"}
for char, repl in replacements.items():
text = text.replace(char, repl)
# Remove diacritics
text = unicodedata.normalize("NFD", text)
text = "".join(c for c in text if unicodedata.category(c) != "Mn")
# Replace non-alphanumeric with hyphens
text = re.sub(r"[^a-z0-9]+", "-", text)
text = re.sub(r"-+", "-", text).strip("-")
return text
@@ -0,0 +1,180 @@
"""ContextRouter Routet Artefakte in den richtigen Wissenskontext.
Bestimmt anhand von Quellkonfiguration und Metadaten, in welchen
Kontext (bahn, dhive, privat) ein Artefakt gehört und berechnet
den Zielpfad im entsprechenden Knowledge-Folder.
Requirements: 1.4, 2.3, 3.2, 3.5, 9.6
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from monorepo.knowledge.ingestion.config import SourceConfig
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
VALID_CONTEXTS = ("bahn", "dhive", "privat")
CATEGORY_FOLDER_MAP: dict[str, str] = {
"meeting": "meetings",
"decision": "decisions",
"project": "projects",
"reference": "references",
"link": "links",
"inbox": "inbox",
}
DEFAULT_FOLDER = "inbox"
# ---------------------------------------------------------------------------
# Data Model
# ---------------------------------------------------------------------------
@dataclass
class RoutingDecision:
"""Result of context routing."""
context: str # bahn, dhive, privat
target_folder: str # knowledge subfolder (meetings, decisions, etc.)
artifact_path: Path # full path for the artifact
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def category_to_folder(category: str) -> str:
"""Map a category name to the corresponding knowledge subfolder.
Known categories are mapped to their plural folder name.
Unknown categories default to 'inbox'.
Args:
category: The category string (e.g. 'meeting', 'decision').
Returns:
The folder name (e.g. 'meetings', 'decisions', 'inbox').
"""
return CATEGORY_FOLDER_MAP.get(category.lower(), DEFAULT_FOLDER)
# ---------------------------------------------------------------------------
# ContextRouter
# ---------------------------------------------------------------------------
class ContextRouter:
"""Routes incoming data to the correct context knowledge folder.
Uses source configuration or working directory to determine the
target context, then computes the full artifact path within the
monorepo structure.
"""
def __init__(self, monorepo_root: Path) -> None:
"""Initialize with the monorepo root path.
Args:
monorepo_root: Absolute path to the monorepo root directory.
"""
self.monorepo_root = monorepo_root
def determine_context(
self,
source_config: SourceConfig | None = None,
cwd: Path | None = None,
) -> str:
"""Determine target context from source config or working directory.
Resolution order:
1. If source_config is provided and has a 'context' param, use that.
2. Otherwise derive from cwd by checking if path contains a known
context segment (/bahn/, /dhive/, /privat/).
3. If neither works, raise ValueError.
Args:
source_config: Optional source configuration with context param.
cwd: Optional working directory to derive context from.
Returns:
The context string ('bahn', 'dhive', or 'privat').
Raises:
ValueError: If context cannot be determined from either source.
"""
# Strategy 1: Extract from source_config params
if source_config is not None:
context_from_config = source_config.params.get("context", "")
if context_from_config and context_from_config in VALID_CONTEXTS:
return context_from_config
# Strategy 2: Derive from working directory path
if cwd is not None:
# Normalize to forward slashes for consistent matching
path_str = str(cwd).replace("\\", "/")
for ctx in VALID_CONTEXTS:
if f"/{ctx}/" in path_str or path_str.endswith(f"/{ctx}"):
return ctx
raise ValueError(
"Kontext konnte nicht bestimmt werden. "
"Weder source_config.params['context'] noch cwd enthalten "
f"einen gültigen Kontext ({', '.join(VALID_CONTEXTS)})."
)
def resolve_target_path(
self, context: str, category: str, filename: str
) -> Path:
"""Compute full path: {root}/{context}/knowledge/{category_folder}/{filename}.
Args:
context: The target context ('bahn', 'dhive', or 'privat').
category: The artifact category (e.g. 'meeting', 'decision').
filename: The filename for the artifact.
Returns:
The full Path for the artifact within the knowledge folder.
"""
folder = category_to_folder(category)
return self.monorepo_root / context / "knowledge" / folder / filename
def route(
self,
category: str,
filename: str,
source_config: SourceConfig | None = None,
cwd: Path | None = None,
) -> RoutingDecision:
"""Full routing: determine context and compute target path.
Convenience method combining determine_context and resolve_target_path.
Args:
category: The artifact category.
filename: The artifact filename.
source_config: Optional source configuration.
cwd: Optional working directory.
Returns:
A RoutingDecision with context, target_folder, and artifact_path.
Raises:
ValueError: If context cannot be determined.
"""
context = self.determine_context(source_config, cwd)
folder = category_to_folder(category)
artifact_path = self.resolve_target_path(context, category, filename)
return RoutingDecision(
context=context,
target_folder=folder,
artifact_path=artifact_path,
)
@@ -0,0 +1,36 @@
"""Externe Integrationen für das Knowledge-Management.
Bindet externe Systeme wie OrgMyLife, Kiro und LiteLLM an die
Ingestion-Pipeline an. Enthält außerdem den Federation-Sync-Filter
für die Knowledge-Ordner-Behandlung bei der Team-Repo-Synchronisation.
"""
from monorepo.knowledge.integrations.federation import (
FederationSyncFilter,
FederationSyncFilterResult,
SyncExclusion,
)
from monorepo.knowledge.integrations.kiro_client import (
KiroClient,
KiroEnrichmentResult,
KiroUnavailableError,
check_kiro_available,
create_kiro_client,
)
from monorepo.knowledge.integrations.orgmylife import (
OrgMyLifeIntegration,
TaskMapping,
)
__all__ = [
"FederationSyncFilter",
"FederationSyncFilterResult",
"KiroClient",
"KiroEnrichmentResult",
"KiroUnavailableError",
"OrgMyLifeIntegration",
"SyncExclusion",
"TaskMapping",
"check_kiro_available",
"create_kiro_client",
]
@@ -0,0 +1,291 @@
"""Federation-Sync-Filter für Knowledge-Artefakte.
Erweitert die bestehende Federation-Logik um Knowledge-Ordner-Behandlung:
- Exkludiert Artefakte mit `shareable: false` aus der Sync-Liste
- Exkludiert ctx-guard-markierte sensible Artefakte
- Stellt sicher, dass keine kontextübergreifenden Links oder Referenzen
auf andere Kontexte in das Team-Repo gelangen
Requirements: 14.1, 14.2, 14.3, 14.4, 14.5
"""
from __future__ import annotations
import logging
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from monorepo.knowledge.artifact import parse_frontmatter, ArtifactMetadata
logger = logging.getLogger(__name__)
# Valid monorepo contexts
VALID_CONTEXTS = {"bahn", "dhive", "privat", "shared"}
# Pattern for cross-context links in frontmatter/content
# Matches references like "bahn/knowledge/...", "dhive/...", "privat/..."
_CROSS_CONTEXT_LINK_PATTERN = re.compile(
r"(?:^|\s|/|\"|\')(?:bahn|dhive|privat|shared)/",
re.MULTILINE,
)
# Pattern for wiki-links referencing other contexts: [[bahn/...]], [[dhive/...]]
_WIKI_LINK_CROSS_CONTEXT = re.compile(
r"\[\[(?:bahn|dhive|privat|shared)/[^\]]+\]\]",
)
@dataclass
class SyncExclusion:
"""Grund für den Ausschluss eines Artefakts von der Synchronisation."""
path: Path
reason: str # "not_shareable", "ctx_guard_sensitive", "cross_context_links"
details: str = ""
@dataclass
class FederationSyncFilterResult:
"""Ergebnis der Federation-Sync-Filterung."""
included: list[Path] = field(default_factory=list)
excluded: list[SyncExclusion] = field(default_factory=list)
@property
def total(self) -> int:
return len(self.included) + len(self.excluded)
class FederationSyncFilter:
"""Filtert Knowledge-Artefakte für die Federation-Synchronisation.
Stellt sicher, dass nur teilbare, nicht-sensible Artefakte ohne
kontextübergreifende Referenzen in das Team-Repo synchronisiert werden.
Die Filterung respektiert die bestehende team-repos.yaml-Konfiguration
und behandelt den Knowledge-Ordner als Teil des Kontextordners
(keine separate Sync-Konfiguration nötig).
"""
def __init__(
self,
monorepo_root: Path,
context: str,
ctx_guard: Any | None = None,
) -> None:
"""Initialisiert den Federation-Sync-Filter.
Args:
monorepo_root: Wurzelverzeichnis des Monorepos.
context: Arbeitskontext (bahn, dhive, privat).
ctx_guard: Optionaler ContextGuard für Sensitivitätsprüfung.
"""
self.monorepo_root = monorepo_root
self.context = context
self.ctx_guard = ctx_guard
self._other_contexts = VALID_CONTEXTS - {context}
def filter_knowledge_artifacts(
self, knowledge_path: Path | None = None
) -> FederationSyncFilterResult:
"""Filtert alle Artefakte im Knowledge-Ordner für die Sync.
Scannt den Knowledge-Ordner des Kontexts und bestimmt für jedes
Artefakt, ob es synchronisiert werden darf.
Args:
knowledge_path: Optionaler Pfad zum Knowledge-Ordner.
Standardmäßig: {monorepo_root}/{context}/knowledge/
Returns:
FederationSyncFilterResult mit included/excluded Listen.
"""
if knowledge_path is None:
knowledge_path = self.monorepo_root / self.context / "knowledge"
result = FederationSyncFilterResult()
if not knowledge_path.exists():
return result
# Scan all markdown files in the knowledge folder
for md_file in knowledge_path.rglob("*.md"):
exclusion = self._check_artifact(md_file)
if exclusion is not None:
result.excluded.append(exclusion)
logger.info(
"Federation-Sync: Artefakt ausgeschlossen: %s (%s)",
md_file.name,
exclusion.reason,
)
else:
result.included.append(md_file)
return result
def get_sync_file_list(
self, knowledge_path: Path | None = None
) -> list[Path]:
"""Gibt die Liste der Dateien zurück, die synchronisiert werden dürfen.
Convenience-Methode, die nur die included-Pfade zurückgibt.
Args:
knowledge_path: Optionaler Pfad zum Knowledge-Ordner.
Returns:
Liste der Pfade, die in das Team-Repo synchronisiert werden dürfen.
"""
result = self.filter_knowledge_artifacts(knowledge_path)
return result.included
def should_sync(self, file_path: Path) -> bool:
"""Prüft ob eine einzelne Datei synchronisiert werden darf.
Args:
file_path: Pfad zur Artefakt-Datei.
Returns:
True wenn das Artefakt synchronisiert werden darf.
"""
return self._check_artifact(file_path) is None
def _check_artifact(self, file_path: Path) -> SyncExclusion | None:
"""Prüft ein einzelnes Artefakt auf Sync-Ausschluss.
Prüft in folgender Reihenfolge:
1. shareable: false → ausschließen
2. ctx-guard-sensibel → ausschließen
3. Kontextübergreifende Links → ausschließen
Args:
file_path: Pfad zur Markdown-Datei.
Returns:
SyncExclusion wenn das Artefakt ausgeschlossen wird, sonst None.
"""
# Nur Markdown-Dateien prüfen
if file_path.suffix.lower() not in (".md", ".markdown"):
return None
# Skip non-artifact files like index.md without frontmatter
try:
artifact = parse_frontmatter(file_path)
except (ValueError, FileNotFoundError, OSError):
# Kein gültiges Frontmatter → als normales Markdown durchlassen
return None
metadata = artifact.metadata
# 1. Check shareable field (Req 14.4)
if not metadata.shareable:
return SyncExclusion(
path=file_path,
reason="not_shareable",
details=f"Artefakt '{metadata.title}' hat shareable: false",
)
# 2. Check ctx-guard sensitivity (Req 14.5)
if self._is_ctx_guard_sensitive(file_path):
return SyncExclusion(
path=file_path,
reason="ctx_guard_sensitive",
details=f"Artefakt '{metadata.title}' ist ctx-guard-geschützt",
)
# 3. Check for cross-context links (Req 14.3)
cross_ctx_issue = self._has_cross_context_references(
artifact.content, metadata
)
if cross_ctx_issue:
return SyncExclusion(
path=file_path,
reason="cross_context_links",
details=cross_ctx_issue,
)
return None
def _is_ctx_guard_sensitive(self, file_path: Path) -> bool:
"""Prüft ob eine Datei über ctx-guard als sensibel markiert ist.
Nutzt den bestehenden ContextGuard-Mechanismus: Wenn ein
externer Kontext (z.B. "shared") keinen Zugriff auf die Datei hat,
gilt sie als sensibel und wird von der Sync ausgeschlossen.
Args:
file_path: Pfad zur Datei.
Returns:
True wenn die Datei sensibel ist.
"""
if self.ctx_guard is None:
return False
# A file is considered sensitive if the "shared" context cannot access it.
# This uses the existing ContextGuard mechanism to determine sensitivity.
try:
return not self.ctx_guard.check_access("shared", file_path)
except (FileNotFoundError, ValueError):
# If ctx-guard can't determine access, assume safe
return False
def _has_cross_context_references(
self, content: str, metadata: ArtifactMetadata
) -> str:
"""Prüft ob ein Artefakt kontextübergreifende Referenzen enthält.
Untersucht sowohl den Content als auch die Frontmatter-Links
auf Verweise auf andere Kontexte.
Args:
content: Markdown-Inhalt des Artefakts.
metadata: Geparste Metadaten.
Returns:
Beschreibung des Problems oder leerer String wenn keine
kontextübergreifenden Referenzen gefunden wurden.
"""
issues: list[str] = []
# Check frontmatter links for cross-context references
for link in metadata.links:
target = link.target
for other_ctx in self._other_contexts:
if target.startswith(f"{other_ctx}/") or f"/{other_ctx}/" in target:
issues.append(
f"Link-Ziel verweist auf Kontext '{other_ctx}': {target}"
)
# Check people references for cross-context paths
for person_ref in metadata.people:
for other_ctx in self._other_contexts:
if f"{other_ctx}/" in person_ref:
issues.append(
f"People-Referenz verweist auf Kontext '{other_ctx}': {person_ref}"
)
# Check project references for cross-context paths
for project_ref in metadata.projects:
for other_ctx in self._other_contexts:
if f"{other_ctx}/" in project_ref:
issues.append(
f"Project-Referenz verweist auf Kontext '{other_ctx}': {project_ref}"
)
# Check content for wiki-links to other contexts
for match in _WIKI_LINK_CROSS_CONTEXT.finditer(content):
link_text = match.group(0)
# Only flag if it references a different context
for other_ctx in self._other_contexts:
if f"[[{other_ctx}/" in link_text:
issues.append(
f"Wiki-Link verweist auf Kontext '{other_ctx}': {link_text}"
)
if issues:
return "; ".join(issues)
return ""
@@ -0,0 +1,316 @@
"""Kiro AI-Agent Client LLM-Provider für Enrichment und OCR.
Stellt das Interface für den Kiro AI-Agent als primären LLM-Provider bereit.
Da Kiro der laufende Orchestrator-Agent selbst ist, arbeitet die reale
Invokation über die Agent-Runtime (kein REST-Call). Diese Klasse definiert
das Interface, das der EnrichmentAgent aufruft, und implementiert die
Fallback-Logik bei Nicht-Verfügbarkeit.
Nutzung:
client = create_kiro_client()
if client is not None:
result = client.enrich_content("Meeting notes ...", "confluence")
text = client.ocr_image(Path("scan.png"))
else:
# Kiro nicht verfügbar → enrichment_pending: true setzen
...
Requirements: 9.3, 9.7
"""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Protocol, runtime_checkable
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Exceptions
# ---------------------------------------------------------------------------
class KiroUnavailableError(Exception):
"""Raised when the Kiro AI-Agent is not available.
This indicates that:
- The Kiro agent session is not active
- The caller should set `enrichment_pending: true` on the artifact
- The artifact can be re-enriched on a subsequent run when Kiro is available
"""
def __init__(self, reason: str = "Kiro AI-Agent session is not active") -> None:
self.reason = reason
super().__init__(reason)
# ---------------------------------------------------------------------------
# Data Classes
# ---------------------------------------------------------------------------
@dataclass
class KiroEnrichmentResult:
"""Result from Kiro-based content enrichment.
Attributes:
title: Generated or extracted title.
category: Content classification (meeting, decision, project, reference, link, inbox).
tags: Generated tags.
people: Detected person names.
projects: Detected project references.
action_items: Extracted action item descriptions.
confidence: Overall confidence score (0.0 - 1.0).
"""
title: str = ""
category: str = "inbox"
tags: list[str] = field(default_factory=list)
people: list[str] = field(default_factory=list)
projects: list[str] = field(default_factory=list)
action_items: list[str] = field(default_factory=list)
confidence: float = 0.0
# ---------------------------------------------------------------------------
# Protocol (Interface Definition)
# ---------------------------------------------------------------------------
@runtime_checkable
class KiroClientProtocol(Protocol):
"""Protocol defining the Kiro client interface.
Any implementation (real agent runtime, mock, or alternative provider)
must satisfy this protocol for the EnrichmentAgent to use it.
"""
def enrich_content(self, content: str, source_type: str = "") -> KiroEnrichmentResult:
"""Analyze content and return enrichment metadata.
Args:
content: The text content to analyze.
source_type: Optional hint about content origin (e.g., "confluence", "email").
Returns:
KiroEnrichmentResult with extracted metadata.
Raises:
KiroUnavailableError: If the Kiro agent is not reachable.
"""
...
def ocr_image(self, image_path: Path) -> str:
"""Extract text from an image using Kiro Vision.
Args:
image_path: Path to the image file (PNG, JPG, JPEG).
Returns:
Extracted text content from the image.
Raises:
KiroUnavailableError: If the Kiro agent is not reachable.
FileNotFoundError: If the image file does not exist.
"""
...
@property
def is_available(self) -> bool:
"""Check if the Kiro agent is currently available.
Returns:
True if the agent session is active and ready.
"""
...
# ---------------------------------------------------------------------------
# Implementation
# ---------------------------------------------------------------------------
class KiroClient:
"""Wrapper for the Kiro AI-Agent as LLM provider.
Since Kiro IS the running agent itself, the actual LLM invocations happen
through the agent runtime (not via REST API). This class defines the
interface and implements availability checking + fallback signaling.
In the current implementation, Kiro availability is determined by an
environment marker or explicit flag. When Kiro is unavailable (e.g., in
CI pipelines or batch processing without an active agent session), all
methods raise KiroUnavailableError so the caller can set
`enrichment_pending: true`.
Usage in the EnrichmentAgent:
client = create_kiro_client()
if client is not None:
try:
result = client.enrich_content(text)
except KiroUnavailableError:
# Set enrichment_pending: true
pass
"""
def __init__(self, *, force_available: bool | None = None) -> None:
"""Initialize the Kiro client.
Args:
force_available: Override availability detection.
- None: auto-detect from environment (default)
- True: force client to report as available (for testing)
- False: force client to report as unavailable
"""
self._force_available = force_available
@property
def is_available(self) -> bool:
"""Check if the Kiro agent is currently available.
Detection strategy:
1. If force_available is set explicitly, use that value.
2. Otherwise, check for KIRO_SESSION_ACTIVE env var.
3. Default: assume NOT available (safe fallback that triggers
enrichment_pending rather than silently skipping enrichment).
Returns:
True if the agent session is active.
"""
if self._force_available is not None:
return self._force_available
import os
# Check environment marker set by the Kiro agent runtime
session_marker = os.environ.get("KIRO_SESSION_ACTIVE", "").lower()
return session_marker in ("1", "true", "yes")
def enrich_content(self, content: str, source_type: str = "") -> KiroEnrichmentResult:
"""Analyze content using the Kiro AI-Agent for enrichment.
When the agent is available, this method delegates to the Kiro runtime
for LLM-based analysis. When unavailable, it raises KiroUnavailableError
to signal that the caller should set enrichment_pending: true.
Args:
content: The text content to analyze.
source_type: Optional source type hint (e.g., "confluence", "jira").
Returns:
KiroEnrichmentResult with LLM-generated metadata.
Raises:
KiroUnavailableError: If the Kiro agent session is not active.
"""
if not self.is_available:
raise KiroUnavailableError(
"Kiro AI-Agent session is not active. "
"Set KIRO_SESSION_ACTIVE=1 or run within an active Kiro session."
)
# --- Agent Runtime Invocation Placeholder ---
# In a live Kiro session, this is where the agent runtime would be
# called to perform LLM-based content analysis. The agent processes
# the content and returns structured enrichment data.
#
# For now, this returns a minimal placeholder result indicating that
# the interface is working but the actual LLM call needs the agent
# runtime context (which is available during interactive sessions).
logger.info(
"Kiro enrich_content called (content_length=%d, source_type=%s)",
len(content),
source_type or "unspecified",
)
return KiroEnrichmentResult(
title="",
category="inbox",
confidence=0.0,
)
def ocr_image(self, image_path: Path) -> str:
"""Extract text from an image using Kiro Vision.
Sends the image to the Kiro agent which uses its vision capabilities
to extract text content. This is the primary OCR method - no external
OCR engine (Tesseract) is required when Kiro is active.
Args:
image_path: Path to the image file.
Returns:
Extracted text from the image.
Raises:
KiroUnavailableError: If the Kiro agent is not active.
FileNotFoundError: If the image file does not exist.
"""
if not image_path.exists():
raise FileNotFoundError(f"Image file not found: {image_path}")
if not self.is_available:
raise KiroUnavailableError(
"Kiro Vision is not available. "
"OCR requires an active Kiro agent session."
)
# --- Agent Runtime Invocation Placeholder ---
# In a live Kiro session, the image would be sent to the agent's
# vision model for text extraction. The agent reads the image and
# returns the extracted text.
logger.info("Kiro ocr_image called (path=%s)", image_path)
return ""
# ---------------------------------------------------------------------------
# Factory Function
# ---------------------------------------------------------------------------
def create_kiro_client(*, force_available: bool | None = None) -> KiroClient | None:
"""Factory function to create a KiroClient if the agent is available.
This is the recommended way to obtain a KiroClient instance. It checks
availability and returns None if Kiro is not active, allowing the caller
to handle the fallback gracefully.
Usage:
client = create_kiro_client()
if client is None:
# Kiro not available → set enrichment_pending: true
artifact.enrichment_pending = True
else:
result = client.enrich_content(content)
Args:
force_available: Override auto-detection (for testing).
Returns:
KiroClient instance if available, None otherwise.
"""
client = KiroClient(force_available=force_available)
if not client.is_available:
logger.debug("Kiro client not available enrichment will be marked as pending")
return None
logger.debug("Kiro client is available")
return client
def check_kiro_available() -> bool:
"""Quick check if Kiro is available without creating a client.
Useful for conditional logic where you just need to know if enrichment
will work, without creating the full client object.
Returns:
True if Kiro agent session is active.
"""
client = KiroClient()
return client.is_available
@@ -0,0 +1,326 @@
"""OrgMyLife-Integration für das Knowledge-Management.
Erstellt Tasks in OrgMyLife aus erkannten Action-Items und verwaltet
die Deduplication über eine Hash-basierte Mapping-Datei.
Requirements: 10.1, 10.2, 10.3, 10.4, 10.5, 10.6
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import date
from pathlib import Path
from typing import Any
import yaml
from monorepo.knowledge.ingestion.enrichment import ActionItem
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Data Classes
# ---------------------------------------------------------------------------
@dataclass
class TaskMapping:
"""Maps artifact action items to OrgMyLife task IDs.
Attributes:
artifact_id: Identifier of the source artifact.
action_item_hash: SHA-256 hash of the action item description.
task_id: OrgMyLife task ID returned by the API.
created: ISO date string when the mapping was created.
"""
artifact_id: str
action_item_hash: str
task_id: int
created: str # ISO date
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
#: Default filename for the task mapping file.
MAPPING_FILENAME = ".task-mapping.yaml"
# ---------------------------------------------------------------------------
# OrgMyLife Integration
# ---------------------------------------------------------------------------
class OrgMyLifeIntegration:
"""Creates tasks in OrgMyLife from detected action items.
Uses a mapping file (`{context}/knowledge/.task-mapping.yaml`) to track
which action items have already been created as tasks, preventing
duplicate task creation on repeated ingestion runs.
The actual OrgMyLife API call is stubbed with a TODO for now.
The integration handles API failures gracefully by logging warnings
and continuing without task creation.
Attributes:
context_path: Path to the context knowledge folder.
enabled: Whether task creation is enabled.
"""
def __init__(self, context_path: Path, enabled: bool = True) -> None:
"""Initialize the OrgMyLife integration.
Args:
context_path: Path to the context knowledge folder
(e.g., `{root}/bahn/knowledge/`).
enabled: Whether task creation is active. If False,
create_tasks() returns an empty list.
"""
self.context_path = context_path
self.enabled = enabled
self._mapping_path = context_path / MAPPING_FILENAME
def create_tasks(
self,
artifact_id: str,
action_items: list[ActionItem],
source_path: str,
) -> list[TaskMapping]:
"""Create OrgMyLife tasks for new action items (dedup via mapping file).
Only creates tasks for action items whose hash is not yet present
in the mapping file. If the OrgMyLife API is unavailable, logs a
warning and returns an empty list.
Args:
artifact_id: Unique identifier of the source artifact.
action_items: List of ActionItem instances to create tasks for.
source_path: Path to the source artifact file (used as source_url).
Returns:
List of TaskMapping instances for newly created tasks.
"""
if not self.enabled:
return []
if not action_items:
return []
# Load existing mappings for deduplication
existing_mappings = self._load_mapping()
new_mappings: list[TaskMapping] = []
for item in action_items:
# Build dedup key: artifact_id + action_item_hash
dedup_key = f"{artifact_id}:{item.hash}"
# Skip if already mapped (Requirement 10.5, 10.6)
if dedup_key in existing_mappings:
logger.debug(
"Action item already mapped: %s (task_id=%d)",
dedup_key,
existing_mappings[dedup_key].task_id,
)
continue
# Create task via OrgMyLife API
task_id = self._create_orgmylife_task(
title=item.description,
source_url=source_path,
deadline=item.deadline,
)
if task_id is None:
# API failure logged in _create_orgmylife_task, continue
continue
# Record new mapping
mapping = TaskMapping(
artifact_id=artifact_id,
action_item_hash=item.hash,
task_id=task_id,
created=date.today().isoformat(),
)
new_mappings.append(mapping)
existing_mappings[dedup_key] = mapping
# Persist updated mappings
if new_mappings:
self._save_mapping(existing_mappings)
logger.info(
"Created %d OrgMyLife task(s) for artifact '%s'",
len(new_mappings),
artifact_id,
)
return new_mappings
def _load_mapping(self) -> dict[str, TaskMapping]:
"""Load task mappings from the YAML file.
Returns:
Dictionary mapping dedup keys (artifact_id:hash) to TaskMapping.
"""
if not self._mapping_path.exists():
return {}
try:
raw = yaml.safe_load(self._mapping_path.read_text(encoding="utf-8"))
except (yaml.YAMLError, OSError) as exc:
logger.warning(
"Failed to load task mapping from %s: %s",
self._mapping_path,
exc,
)
return {}
if not raw or not isinstance(raw, dict):
return {}
mappings: dict[str, TaskMapping] = {}
entries = raw.get("mappings", [])
if not isinstance(entries, list):
return {}
for entry in entries:
if not isinstance(entry, dict):
continue
try:
mapping = TaskMapping(
artifact_id=str(entry["artifact_id"]),
action_item_hash=str(entry["action_item_hash"]),
task_id=int(entry["task_id"]),
created=str(entry.get("created", "")),
)
dedup_key = f"{mapping.artifact_id}:{mapping.action_item_hash}"
mappings[dedup_key] = mapping
except (KeyError, ValueError, TypeError) as exc:
logger.warning("Skipping invalid mapping entry: %s", exc)
continue
return mappings
def _save_mapping(self, mappings: dict[str, TaskMapping]) -> None:
"""Save task mappings to the YAML file.
Args:
mappings: Dictionary of dedup keys to TaskMapping instances.
"""
entries: list[dict[str, Any]] = []
for mapping in mappings.values():
entries.append({
"artifact_id": mapping.artifact_id,
"action_item_hash": mapping.action_item_hash,
"task_id": mapping.task_id,
"created": mapping.created,
})
data = {
"version": "1.0",
"mappings": entries,
}
try:
self._mapping_path.parent.mkdir(parents=True, exist_ok=True)
self._mapping_path.write_text(
yaml.dump(data, default_flow_style=False, allow_unicode=True),
encoding="utf-8",
)
except OSError as exc:
logger.warning(
"Failed to save task mapping to %s: %s",
self._mapping_path,
exc,
)
def _create_orgmylife_task(
self,
title: str,
source_url: str,
deadline: str | None = None,
) -> int | None:
"""Create a task in OrgMyLife via the REST API.
This method handles API errors gracefully: on failure it logs a
warning and returns None (Requirement 10.4).
Args:
title: Task title (action item description).
source_url: Reference to the source artifact.
deadline: Optional ISO date string for the task deadline.
Returns:
The OrgMyLife task ID on success, or None on failure.
"""
# TODO: Implement actual OrgMyLife REST API call.
# The OrgMyLife API endpoint for task creation:
# POST /api/tasks
# Body: {"title": title, "repo": "", "deadline": deadline}
# Response: {"id": <int>, "title": ..., ...}
#
# For now, use a stub that simulates a successful API call.
# In production, replace with actual HTTP client (httpx/requests).
try:
task_id = self._call_orgmylife_api(title, source_url, deadline)
return task_id
except Exception as exc: # noqa: BLE001
# Graceful error handling (Requirement 10.4)
logger.warning(
"OrgMyLife API error creating task '%s': %s",
title,
exc,
)
return None
def _call_orgmylife_api(
self,
title: str,
source_url: str,
deadline: str | None = None,
) -> int:
"""Actually call the OrgMyLife API.
This is separated from _create_orgmylife_task to allow easy
mocking in tests and future replacement with real HTTP calls.
Args:
title: Task title.
source_url: Reference to source artifact.
deadline: Optional deadline ISO date string.
Returns:
The created task ID.
Raises:
RuntimeError: If the API call fails.
"""
# TODO: Replace with actual HTTP call to OrgMyLife API.
# Example implementation with httpx:
#
# import httpx
# response = httpx.post(
# f"{ORGMYLIFE_BASE_URL}/api/tasks",
# json={
# "title": title,
# "repo": "",
# "source_url": source_url,
# "deadline": deadline,
# },
# timeout=10.0,
# )
# response.raise_for_status()
# return response.json()["id"]
# Stub: raise to signal that real API is not implemented yet.
# In tests, this method is patched to return a task ID.
raise NotImplementedError(
"OrgMyLife API not yet connected. "
"Patch _call_orgmylife_api in tests or implement HTTP client."
)
@@ -4,8 +4,32 @@ Jede Quellstrategie implementiert das Extrahieren von Wissensartefakten
aus einem bestimmten Format/Medium (Markdown, Confluence, PDF, etc.).
"""
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceConfig,
SourceError,
SourceStrategy,
)
from monorepo.knowledge.sources.chat import ChatSource
from monorepo.knowledge.sources.confluence import ConfluenceSource
from monorepo.knowledge.sources.email import EmailSource
from monorepo.knowledge.sources.jira import JiraSource
from monorepo.knowledge.sources.link import LinkRegistry
from monorepo.knowledge.sources.markdown import MarkdownSource
from monorepo.knowledge.sources.pdf import PDFSource
from monorepo.knowledge.sources.wissensdatenbank import WissensdatenbankSource
__all__ = [
"ChatSource",
"ConfluenceSource",
"EmailSource",
"ExtractionResult",
"JiraSource",
"LinkRegistry",
"MarkdownSource",
"PDFSource",
"SourceConfig",
"SourceError",
"SourceStrategy",
"WissensdatenbankSource",
]
@@ -0,0 +1,185 @@
"""Abstrakte Basisklasse für Quellstrategien der ETL-Pipeline.
Definiert das SourceStrategy-Interface, das alle konkreten Quellstrategien
(Markdown, PDF, Confluence, Jira, etc.) implementieren müssen.
Requirements: 2.2, 2.4, 2.7, 4.7
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from pathlib import PurePosixPath, PureWindowsPath
from typing import Any
from monorepo.knowledge.artifact import Artifact
# ---------------------------------------------------------------------------
# Format Detection (Requirement 4.7)
# ---------------------------------------------------------------------------
#: Mapping von unterstützten Dateiendungen (ohne Punkt, lowercase) auf Formattypen.
EXTENSION_FORMAT_MAP: dict[str, str] = {
"md": "markdown",
"txt": "text",
"pdf": "pdf",
"png": "image",
"jpg": "image",
"jpeg": "image",
"docx": "docx",
}
#: Menge aller unterstützten Endungen (lowercase, ohne Punkt).
SUPPORTED_EXTENSIONS: set[str] = set(EXTENSION_FORMAT_MAP.keys())
def detect_format(file_path: str) -> str:
"""Erkennt den Formattyp anhand der Dateiendung.
Unterstützte Zuordnungen:
- .md → "markdown"
- .txt → "text"
- .pdf → "pdf"
- .png, .jpg, .jpeg → "image"
- .docx → "docx"
Args:
file_path: Ein Dateipfad (kann relativ oder absolut sein).
Returns:
Der erkannte Formattyp als String.
Raises:
ValueError: Falls die Dateiendung nicht unterstützt wird.
"""
# Support both Unix and Windows paths
# Use suffix extraction that handles both separators
path_str = file_path.strip()
if not path_str:
raise ValueError("Empty file path")
# Extract extension: find the last '.' in the filename portion
# Handle both forward and backslash separators
filename = path_str.replace("\\", "/").rsplit("/", 1)[-1]
if "." not in filename or filename.startswith("."):
raise ValueError(
f"Unsupported or missing file extension in: {file_path!r}"
)
ext = filename.rsplit(".", 1)[-1].lower()
if ext in EXTENSION_FORMAT_MAP:
return EXTENSION_FORMAT_MAP[ext]
raise ValueError(
f"Unsupported file extension '.{ext}' in: {file_path!r}. "
f"Supported: {', '.join('.' + e for e in sorted(SUPPORTED_EXTENSIONS))}"
)
# ---------------------------------------------------------------------------
# Konfiguration
# ---------------------------------------------------------------------------
@dataclass
class SourceConfig:
"""Konfiguration für eine einzelne Quelle in sources.yaml.
Attributes:
type: Quelltyp (confluence, jira, email, file, link, wissensdatenbank).
name: Menschenlesbare Bezeichnung der Quelle.
params: Typspezifische Verbindungs-/Filterparameter.
target_folder: Optionaler Override für den Zielordner.
sync_frequency: Synchronisierungsfrequenz (on-push, hourly, daily, manual).
enabled: Ob die Quelle aktiv ist.
"""
type: str
name: str
params: dict[str, Any] = field(default_factory=dict)
target_folder: str = ""
sync_frequency: str = "manual"
enabled: bool = True
# ---------------------------------------------------------------------------
# Ergebnisse und Fehler
# ---------------------------------------------------------------------------
@dataclass
class SourceError:
"""Strukturierter Fehler bei der Quellextraktion.
Attributes:
source_name: Bezeichnung der betroffenen Quelle.
error_type: Fehlerkategorie (connection, auth, parse, write, llm).
message: Menschenlesbare Fehlerbeschreibung.
timestamp: Zeitstempel im ISO-8601-Format.
retry: Ob ein Wiederholungsversuch sinnvoll ist.
artifact_path: Betroffener Artefaktpfad, falls zutreffend.
"""
source_name: str
error_type: str
message: str
timestamp: str
retry: bool
artifact_path: str = ""
@dataclass
class ExtractionResult:
"""Ergebnis einer Quellstrategie-Extraktion.
Attributes:
artifacts: Erfolgreich extrahierte Artefakte.
errors: Fehler, die bei der Extraktion aufgetreten sind.
skipped: Anzahl übersprungener Elemente (z.B. unveränderte Dateien).
"""
artifacts: list[Artifact] = field(default_factory=list)
errors: list[SourceError] = field(default_factory=list)
skipped: int = 0
# ---------------------------------------------------------------------------
# Abstrakte Basisklasse
# ---------------------------------------------------------------------------
class SourceStrategy(ABC):
"""Abstrakte Basisklasse für alle Quellstrategien.
Jede konkrete Quellstrategie (Markdown, PDF, Confluence, etc.) muss
dieses Interface implementieren, um in die Ingestion-Pipeline integriert
werden zu können.
"""
@abstractmethod
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
"""Extrahiert Rohartefakte aus der konfigurierten Quelle.
Args:
config: Die Quellkonfiguration mit Verbindungsparametern.
context: Der Zielkontext (bahn, dhive, privat).
Returns:
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
"""
...
@abstractmethod
def supports_incremental(self) -> bool:
"""Gibt an, ob diese Strategie inkrementelle Verarbeitung unterstützt.
Bei inkrementeller Verarbeitung werden nur neue oder geänderte
Inhalte verarbeitet (z.B. über Content-Hashes).
Returns:
True wenn inkrementelle Verarbeitung unterstützt wird.
"""
...
@@ -0,0 +1,365 @@
"""Chat-Quellstrategie für die ETL-Pipeline.
Verarbeitet Chat-Exporte in Text- oder JSON-Format und erstellt
strukturierte Markdown-Artefakte mit Frontmatter.
Unterstützte Formate:
- Plain-Text mit Timestamp-Patterns (z.B. "[2025-01-15 14:30] User: Message")
- JSON-Arrays mit Nachrichtenobjekten (sender, timestamp, text)
Requirements: 7.3, 7.5
"""
from __future__ import annotations
import hashlib
import json
import logging
import re
from dataclasses import dataclass
from datetime import date, datetime
from pathlib import Path
from typing import Any
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceConfig,
SourceError,
SourceStrategy,
)
logger = logging.getLogger(__name__)
__all__ = ["ChatSource"]
# Supported chat export file extensions
_CHAT_EXTENSIONS = {".txt", ".json"}
# Patterns for detecting timestamped chat messages in plain text
# Format: [YYYY-MM-DD HH:MM] or [YYYY-MM-DD HH:MM:SS] followed by sender: message
_TIMESTAMP_PATTERNS = [
# [2025-01-15 14:30] User: Message
re.compile(
r"^\[(\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}(?::\d{2})?)\]\s*(.+?):\s*(.+)$",
re.MULTILINE,
),
# 2025-01-15 14:30 - User: Message (WhatsApp-style)
re.compile(
r"^(\d{4}-\d{2}-\d{2}[\s,]+\d{2}:\d{2}(?::\d{2})?)\s*-\s*(.+?):\s*(.+)$",
re.MULTILINE,
),
# DD.MM.YYYY, HH:MM - User: Message (German WhatsApp)
re.compile(
r"^(\d{2}\.\d{2}\.\d{4},?\s*\d{2}:\d{2}(?::\d{2})?)\s*-\s*(.+?):\s*(.+)$",
re.MULTILINE,
),
]
# ---------------------------------------------------------------------------
# Helper functions
# ---------------------------------------------------------------------------
def _slugify(text: str) -> str:
"""Creates a URL-friendly slug from text."""
slug = text.lower().strip()
slug = re.sub(r"[^a-z0-9äöüß\s-]", "", slug)
slug = re.sub(r"[\s_]+", "-", slug)
slug = re.sub(r"-+", "-", slug)
slug = slug.strip("-")
return slug[:80] if slug else "chat-export"
def _compute_content_hash(content: str) -> str:
"""Computes SHA-256 hash of content."""
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
return f"sha256:{digest}"
def _parse_date_str(date_str: str) -> datetime | None:
"""Attempts to parse various date formats into a datetime."""
formats = [
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d %H:%M",
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%dT%H:%M",
"%d.%m.%Y, %H:%M:%S",
"%d.%m.%Y, %H:%M",
"%d.%m.%Y %H:%M:%S",
"%d.%m.%Y %H:%M",
]
date_str = date_str.strip()
for fmt in formats:
try:
return datetime.strptime(date_str, fmt)
except ValueError:
continue
return None
@dataclass
class ChatMessage:
"""A single parsed chat message."""
timestamp: datetime | None
sender: str
text: str
# ---------------------------------------------------------------------------
# ChatSource implementiert SourceStrategy ABC
# ---------------------------------------------------------------------------
class ChatSource(SourceStrategy):
"""Quellstrategie: Verarbeitet Chat-Exporte (Text/JSON) aus einem Verzeichnis.
Unterstützt:
- Plain-Text-Exporte mit erkennbaren Timestamp-Patterns
- JSON-Exporte als Array von Message-Objekten
Erstellt pro Exportdatei ein Markdown-Artefakt mit Konversationsinhalt
und Frontmatter-Metadaten (Teilnehmer, Zeitraum, Quelltyp).
"""
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
"""Extrahiert Chat-Artefakte aus dem konfigurierten Verzeichnis.
Args:
config: Quellkonfiguration mit params["directory"] als Pfad.
context: Zielkontext (bahn, dhive, privat).
Returns:
ExtractionResult mit extrahierten Artefakten und Fehlern.
"""
directory = self._resolve_directory(config)
artifacts: list[Artifact] = []
errors: list[SourceError] = []
if not directory.exists():
errors.append(SourceError(
source_name=config.name,
error_type="parse",
message=f"Verzeichnis existiert nicht: {directory}",
timestamp="",
retry=True,
))
return ExtractionResult(artifacts=artifacts, errors=errors)
if not directory.is_dir():
errors.append(SourceError(
source_name=config.name,
error_type="parse",
message=f"Pfad ist kein Verzeichnis: {directory}",
timestamp="",
retry=False,
))
return ExtractionResult(artifacts=artifacts, errors=errors)
# Find all chat export files
chat_files = sorted(
f for f in directory.rglob("*")
if f.is_file() and f.suffix.lower() in _CHAT_EXTENSIONS
)
for file_path in chat_files:
try:
artifact = self._process_chat_file(file_path, config.name)
if artifact is not None:
artifacts.append(artifact)
except Exception as e:
logger.warning("Fehler beim Verarbeiten von %s: %s", file_path, e)
errors.append(SourceError(
source_name=config.name,
error_type="parse",
message=f"Fehler bei {file_path.name}: {e}",
timestamp="",
retry=False,
artifact_path=str(file_path),
))
return ExtractionResult(artifacts=artifacts, errors=errors)
def supports_incremental(self) -> bool:
"""Chat-Exporte unterstützen keine inkrementelle Verarbeitung."""
return False
# ------------------------------------------------------------------
# Internal methods
# ------------------------------------------------------------------
def _resolve_directory(self, config: SourceConfig) -> Path:
"""Resolves the source directory from config."""
dir_param = config.params.get("directory", "")
if dir_param:
return Path(dir_param)
raise ValueError(
"ChatSource benötigt params['directory'] in der Konfiguration."
)
def _process_chat_file(self, file_path: Path, source_name: str) -> Artifact | None:
"""Processes a single chat export file."""
suffix = file_path.suffix.lower()
if suffix == ".json":
return self._process_json_chat(file_path, source_name)
elif suffix == ".txt":
return self._process_text_chat(file_path, source_name)
return None
def _process_json_chat(self, file_path: Path, source_name: str) -> Artifact | None:
"""Processes a JSON chat export.
Expected format: array of objects with keys:
- sender (str): message author
- timestamp (str): ISO-format or similar date string
- text (str): message content
"""
raw_text = file_path.read_text(encoding="utf-8")
data = json.loads(raw_text)
if not isinstance(data, list):
logger.warning("JSON-Chat %s ist kein Array, überspringe.", file_path)
return None
messages: list[ChatMessage] = []
for entry in data:
if not isinstance(entry, dict):
continue
sender = str(entry.get("sender", entry.get("author", entry.get("from", ""))))
text = str(entry.get("text", entry.get("message", entry.get("content", ""))))
ts_str = str(entry.get("timestamp", entry.get("date", entry.get("time", ""))))
timestamp = _parse_date_str(ts_str) if ts_str else None
if text:
messages.append(ChatMessage(timestamp=timestamp, sender=sender, text=text))
if not messages:
return None
return self._build_artifact(file_path, messages, source_name, "json")
def _process_text_chat(self, file_path: Path, source_name: str) -> Artifact | None:
"""Processes a plain-text chat export with timestamp patterns."""
raw_text = file_path.read_text(encoding="utf-8")
messages: list[ChatMessage] = []
# Try each pattern until one matches
for pattern in _TIMESTAMP_PATTERNS:
matches = pattern.findall(raw_text)
if matches:
for match in matches:
ts_str, sender, text = match
timestamp = _parse_date_str(ts_str)
messages.append(ChatMessage(
timestamp=timestamp,
sender=sender.strip(),
text=text.strip(),
))
break
# If no pattern matched, treat entire file as a single message block
if not messages:
# Still create an artifact from the raw content
messages.append(ChatMessage(
timestamp=None,
sender="unknown",
text=raw_text.strip(),
))
return self._build_artifact(file_path, messages, source_name, "text")
def _build_artifact(
self,
file_path: Path,
messages: list[ChatMessage],
source_name: str,
source_format: str,
) -> Artifact:
"""Builds a Markdown artifact from parsed chat messages."""
# Collect participants and date range
participants: set[str] = set()
dates: list[datetime] = []
for msg in messages:
if msg.sender and msg.sender != "unknown":
participants.add(msg.sender)
if msg.timestamp:
dates.append(msg.timestamp)
date_start = min(dates).date() if dates else None
date_end = max(dates).date() if dates else None
# Build Markdown content
title = f"Chat: {file_path.stem}"
content_lines: list[str] = []
content_lines.append(f"# {title}\n")
content_lines.append("")
if participants:
content_lines.append(f"**Teilnehmer:** {', '.join(sorted(participants))} ")
if date_start and date_end:
if date_start == date_end:
content_lines.append(f"**Datum:** {date_start.isoformat()} ")
else:
content_lines.append(
f"**Zeitraum:** {date_start.isoformat()} bis {date_end.isoformat()} "
)
content_lines.append("")
content_lines.append("---\n")
content_lines.append("")
# Render messages
for msg in messages:
ts_prefix = ""
if msg.timestamp:
ts_prefix = f"[{msg.timestamp.strftime('%Y-%m-%d %H:%M')}] "
sender_prefix = f"**{msg.sender}:** " if msg.sender and msg.sender != "unknown" else ""
content_lines.append(f"{ts_prefix}{sender_prefix}{msg.text} ")
content = "\n".join(content_lines)
# Build source dict for frontmatter
source_dict: dict[str, str] = {
"type": "chat",
"file": str(file_path),
"format": source_format,
}
# Date range for frontmatter
date_range = ""
if date_start and date_end:
date_range = f"{date_start.isoformat()}/{date_end.isoformat()}"
elif date_start:
date_range = date_start.isoformat()
if date_range:
source_dict["date_range"] = date_range
# Build metadata
metadata = ArtifactMetadata(
type="chat",
title=title,
tags=["chat"],
source_context="",
created=date_start,
updated=date_end,
shareable=False,
source=source_dict,
category="inbox",
people=sorted(participants),
content_hash=_compute_content_hash(content),
)
# Generate filename
date_prefix = date_start.isoformat() if date_start else "undated"
slug = _slugify(file_path.stem)
filename = f"{date_prefix}-{slug}.md"
return Artifact(
metadata=metadata,
content=content,
file_path=Path(filename),
)
@@ -0,0 +1,686 @@
"""Confluence-Quellstrategie für die ETL-Pipeline.
Implementiert die Anbindung an die Confluence REST API (Cloud und Server):
- Einzelne Seiten abrufen
- Seitenbäume rekursiv traversieren
- FAQ-Spaces importieren
- Inkrementelle Updates via Versions-Nummer
- Graceful Error Handling (skip source, log error)
Seiteninhalte werden als Markdown extrahiert und mit reichhaltigem
Frontmatter angereichert (URL, Space, Titel, Last-Modified).
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
"""
from __future__ import annotations
import hashlib
import json
import logging
import re
import urllib.error
import urllib.request
from base64 import b64encode
from dataclasses import dataclass
from datetime import date, datetime
from html import unescape
from pathlib import Path
from typing import Any
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceConfig,
SourceError,
SourceStrategy,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# HTML → Markdown Konvertierung (leichtgewichtig, kein externes Paket)
# ---------------------------------------------------------------------------
def _html_to_markdown(html: str) -> str:
"""Konvertiert einfaches HTML in Markdown.
Unterstützt die gängigsten Confluence-HTML-Elemente:
- Überschriften (h1-h6)
- Absätze, Zeilenumbrüche
- Listen (ul/ol/li)
- Fett/Kursiv
- Links
- Code-Blöcke und Inline-Code
- Tabellen (vereinfacht)
Args:
html: HTML-String aus Confluence Storage Format.
Returns:
Markdown-String.
"""
if not html:
return ""
text = html
# Entferne CDATA und XML-Deklarationen
text = re.sub(r"<!\[CDATA\[.*?\]\]>", "", text, flags=re.DOTALL)
# Code-Blöcke zuerst (vor allgemeinem Tag-Stripping)
text = re.sub(
r"<ac:structured-macro[^>]*ac:name=\"code\"[^>]*>.*?"
r"<ac:plain-text-body>(.*?)</ac:plain-text-body>.*?"
r"</ac:structured-macro>",
r"\n```\n\1\n```\n",
text,
flags=re.DOTALL,
)
# Confluence-spezifische Macros entfernen
text = re.sub(
r"<ac:structured-macro[^>]*>.*?</ac:structured-macro>",
"",
text,
flags=re.DOTALL,
)
text = re.sub(r"<ac:[^>]*/>", "", text)
text = re.sub(r"<ac:[^>]*>.*?</ac:[^>]*>", "", text, flags=re.DOTALL)
text = re.sub(r"<ri:[^>]*/>", "", text)
text = re.sub(r"<ri:[^>]*>.*?</ri:[^>]*>", "", text, flags=re.DOTALL)
# Überschriften
for level in range(1, 7):
prefix = "#" * level
text = re.sub(
rf"<h{level}[^>]*>(.*?)</h{level}>",
rf"\n{prefix} \1\n",
text,
flags=re.DOTALL,
)
# Links
text = re.sub(
r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>',
r"[\2](\1)",
text,
flags=re.DOTALL,
)
# Fett und Kursiv
text = re.sub(r"<strong[^>]*>(.*?)</strong>", r"**\1**", text, flags=re.DOTALL)
text = re.sub(r"<b[^>]*>(.*?)</b>", r"**\1**", text, flags=re.DOTALL)
text = re.sub(r"<em[^>]*>(.*?)</em>", r"*\1*", text, flags=re.DOTALL)
text = re.sub(r"<i[^>]*>(.*?)</i>", r"*\1*", text, flags=re.DOTALL)
# Code (inline)
text = re.sub(r"<code[^>]*>(.*?)</code>", r"`\1`", text, flags=re.DOTALL)
text = re.sub(r"<pre[^>]*>(.*?)</pre>", r"\n```\n\1\n```\n", text, flags=re.DOTALL)
# Listen
text = re.sub(r"<ul[^>]*>", "\n", text)
text = re.sub(r"</ul>", "\n", text)
text = re.sub(r"<ol[^>]*>", "\n", text)
text = re.sub(r"</ol>", "\n", text)
text = re.sub(r"<li[^>]*>(.*?)</li>", r"- \1\n", text, flags=re.DOTALL)
# Absätze und Zeilenumbrüche
text = re.sub(r"<br\s*/?>", "\n", text)
text = re.sub(r"<p[^>]*>(.*?)</p>", r"\1\n\n", text, flags=re.DOTALL)
text = re.sub(r"<div[^>]*>(.*?)</div>", r"\1\n", text, flags=re.DOTALL)
# Tabellen (vereinfachtes Markdown)
text = re.sub(r"<table[^>]*>", "\n", text)
text = re.sub(r"</table>", "\n", text)
text = re.sub(r"<thead[^>]*>|</thead>", "", text)
text = re.sub(r"<tbody[^>]*>|</tbody>", "", text)
text = re.sub(r"<tr[^>]*>(.*?)</tr>", r"|\1\n", text, flags=re.DOTALL)
text = re.sub(r"<th[^>]*>(.*?)</th>", r" \1 |", text, flags=re.DOTALL)
text = re.sub(r"<td[^>]*>(.*?)</td>", r" \1 |", text, flags=re.DOTALL)
# Alle verbleibenden HTML-Tags entfernen
text = re.sub(r"<[^>]+>", "", text)
# HTML-Entities dekodieren
text = unescape(text)
# Mehrfache Leerzeilen reduzieren
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
# ---------------------------------------------------------------------------
# Confluence API Client
# ---------------------------------------------------------------------------
@dataclass
class ConfluencePage:
"""Repräsentiert eine einzelne Confluence-Seite."""
page_id: str
title: str
space_key: str
version: int
last_modified: str
content_html: str
url: str
attachments: list[dict[str, str]]
class ConfluenceClient:
"""Leichtgewichtiger Client für die Confluence REST API.
Verwendet urllib aus der Standardbibliothek (keine requests-Abhängigkeit).
Unterstützt Token-basierte Authentifizierung (Bearer oder Basic mit API-Token).
"""
def __init__(self, base_url: str, token: str, username: str = "") -> None:
"""Initialisiert den Confluence-Client.
Args:
base_url: Basis-URL der Confluence-Instanz (z.B. https://confluence.example.com).
token: API-Token für die Authentifizierung.
username: Benutzername für Basic-Auth (optional, für Cloud).
"""
self.base_url = base_url.rstrip("/")
self.token = token
self.username = username
def _build_auth_header(self) -> str:
"""Erstellt den Authorization-Header.
Returns:
Authorization-Header-Wert.
"""
if self.username:
# Cloud: Basic Auth mit E-Mail + API-Token
credentials = f"{self.username}:{self.token}"
encoded = b64encode(credentials.encode()).decode()
return f"Basic {encoded}"
# Server/DC: Bearer Token
return f"Bearer {self.token}"
def _request(self, path: str) -> dict[str, Any]:
"""Führt einen GET-Request gegen die Confluence REST API durch.
Args:
path: API-Pfad (relativ zu base_url).
Returns:
JSON-Response als Dictionary.
Raises:
urllib.error.URLError: Bei Netzwerkfehlern.
urllib.error.HTTPError: Bei HTTP-Fehlern (401, 403, 404, etc.).
"""
url = f"{self.base_url}{path}"
req = urllib.request.Request(url)
req.add_header("Authorization", self._build_auth_header())
req.add_header("Accept", "application/json")
with urllib.request.urlopen(req, timeout=30) as response:
return json.loads(response.read().decode("utf-8"))
def get_page(self, page_id: str) -> ConfluencePage:
"""Ruft eine einzelne Confluence-Seite ab.
Args:
page_id: Die ID der Seite.
Returns:
ConfluencePage mit Inhalt und Metadaten.
"""
path = (
f"/rest/api/content/{page_id}"
f"?expand=body.storage,version,space,metadata.labels"
f"&expand=children.attachment"
)
data = self._request(path)
return self._parse_page_response(data)
def get_child_pages(self, page_id: str) -> list[str]:
"""Ruft die IDs aller direkten Kindseiten ab.
Args:
page_id: Die ID der Elternseite.
Returns:
Liste von Seiten-IDs.
"""
path = f"/rest/api/content/{page_id}/child/page?limit=200"
data = self._request(path)
results = data.get("results", [])
return [str(r["id"]) for r in results]
def get_space_pages(self, space_key: str) -> list[str]:
"""Ruft alle Seiten-IDs eines Spaces ab.
Args:
space_key: Der Space-Key (z.B. "ACV2").
Returns:
Liste von Seiten-IDs.
"""
page_ids: list[str] = []
start = 0
limit = 50
while True:
path = (
f"/rest/api/content"
f"?spaceKey={space_key}&type=page"
f"&start={start}&limit={limit}"
)
data = self._request(path)
results = data.get("results", [])
page_ids.extend(str(r["id"]) for r in results)
# Paginierung
size = data.get("size", 0)
if size < limit:
break
start += limit
return page_ids
def get_page_tree(self, root_page_id: str) -> list[str]:
"""Traversiert einen Seitenbaum rekursiv und sammelt alle Seiten-IDs.
Args:
root_page_id: Die ID der Wurzelseite.
Returns:
Liste aller Seiten-IDs im Baum (inklusive Wurzel).
"""
collected: list[str] = [root_page_id]
queue = [root_page_id]
while queue:
current_id = queue.pop(0)
children = self.get_child_pages(current_id)
for child_id in children:
if child_id not in collected:
collected.append(child_id)
queue.append(child_id)
return collected
def _parse_page_response(self, data: dict[str, Any]) -> ConfluencePage:
"""Parst die API-Antwort in ein ConfluencePage-Objekt.
Args:
data: JSON-Response der Confluence API.
Returns:
ConfluencePage mit extrahierten Daten.
"""
page_id = str(data.get("id", ""))
title = data.get("title", "Untitled")
# Space
space = data.get("space", {})
space_key = space.get("key", "")
# Version
version_info = data.get("version", {})
version = version_info.get("number", 1)
last_modified = version_info.get("when", "")
# Content
body = data.get("body", {})
storage = body.get("storage", {})
content_html = storage.get("value", "")
# URL
base_url = self.base_url
links = data.get("_links", {})
web_ui = links.get("webui", f"/pages/viewpage.action?pageId={page_id}")
url = f"{base_url}{web_ui}"
# Attachments
attachments: list[dict[str, str]] = []
children = data.get("children", {})
attachment_data = children.get("attachment", {})
for att in attachment_data.get("results", []):
att_title = att.get("title", "")
att_links = att.get("_links", {})
att_download = att_links.get("download", "")
if att_title:
attachments.append({
"title": att_title,
"url": f"{base_url}{att_download}" if att_download else "",
})
return ConfluencePage(
page_id=page_id,
title=title,
space_key=space_key,
version=version,
last_modified=last_modified,
content_html=content_html,
url=url,
attachments=attachments,
)
# ---------------------------------------------------------------------------
# Confluence Source Strategy
# ---------------------------------------------------------------------------
class ConfluenceSource(SourceStrategy):
"""Quellstrategie: Confluence REST API.
Unterstützt drei Abrufmodi:
1. Einzelne Seiten (params.pages: list[str] mit Page-IDs)
2. Seitenbäume (params.tree_roots: list[str] mit Root-Page-IDs)
3. Spaces (params.spaces: list[str] mit Space-Keys)
Inkrementelle Updates werden über die Confluence-Versionsnummer gesteuert.
Bekannte Versionen können über set_known_versions() gesetzt werden.
Fehler bei der API-Kommunikation werden graceful behandelt:
- Einzelne fehlgeschlagene Seiten werden übersprungen
- Bei kompletten API-Ausfällen wird die gesamte Quelle übersprungen
- Bestehende Artefakte werden niemals gelöscht
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7
"""
def __init__(self) -> None:
"""Initialisiert die ConfluenceSource."""
self._known_versions: dict[str, int] = {}
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
"""Extrahiert Artefakte aus Confluence.
Args:
config: SourceConfig mit Confluence-Verbindungsparametern.
context: Der Zielkontext (bahn, dhive, privat).
Returns:
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
"""
artifacts: list[Artifact] = []
errors: list[SourceError] = []
skipped = 0
# Client initialisieren
try:
client = self._create_client(config)
except ValueError as e:
logger.error("Confluence-Konfigurationsfehler: %s", e)
errors.append(SourceError(
source_name=config.name,
error_type="auth",
message=str(e),
timestamp=datetime.now().isoformat(),
retry=False,
))
return ExtractionResult(artifacts=[], errors=errors, skipped=0)
# Seiten-IDs sammeln
page_ids = self._collect_page_ids(client, config, errors)
# Seiten abrufen und verarbeiten
for page_id in page_ids:
try:
page = client.get_page(page_id)
# Inkrementelle Verarbeitung: Version-Check
known_version = self._known_versions.get(page_id, 0)
if known_version >= page.version:
skipped += 1
continue
artifact = self._page_to_artifact(page, context)
artifacts.append(artifact)
except (urllib.error.URLError, urllib.error.HTTPError) as e:
logger.warning(
"Seite %s konnte nicht abgerufen werden: %s", page_id, e
)
errors.append(SourceError(
source_name=config.name,
error_type="connection",
message=f"Seite {page_id}: {e}",
timestamp=datetime.now().isoformat(),
retry=True,
artifact_path=f"confluence:{page_id}",
))
except (json.JSONDecodeError, KeyError, TypeError) as e:
logger.warning(
"Seite %s konnte nicht geparst werden: %s", page_id, e
)
errors.append(SourceError(
source_name=config.name,
error_type="parse",
message=f"Seite {page_id}: {e}",
timestamp=datetime.now().isoformat(),
retry=False,
artifact_path=f"confluence:{page_id}",
))
return ExtractionResult(artifacts=artifacts, errors=errors, skipped=skipped)
def supports_incremental(self) -> bool:
"""Confluence unterstützt inkrementelle Verarbeitung via Versions-Nummer.
Returns:
True inkrementelle Verarbeitung wird unterstützt.
"""
return True
def set_known_versions(self, versions: dict[str, int]) -> None:
"""Setzt bekannte Seitenversionen für inkrementelle Verarbeitung.
Args:
versions: Mapping von Page-ID zu bekannter Versionsnummer.
Seiten mit gleicher oder niedrigerer Version werden übersprungen.
"""
self._known_versions = dict(versions)
# ------------------------------------------------------------------
# Interne Hilfsmethoden
# ------------------------------------------------------------------
def _create_client(self, config: SourceConfig) -> ConfluenceClient:
"""Erstellt einen ConfluenceClient aus der SourceConfig.
Args:
config: SourceConfig mit Confluence-Parametern.
Returns:
Konfigurierter ConfluenceClient.
Raises:
ValueError: Bei fehlenden Pflichtparametern.
"""
params = config.params
base_url = params.get("base_url", "")
if not base_url:
raise ValueError("Confluence base_url ist nicht konfiguriert")
# Token kann unter verschiedenen Keys liegen
token = params.get("token", "") or params.get("api_key", "")
if not token:
raise ValueError(
"Confluence Token fehlt (weder 'token' noch 'api_key' in params)"
)
username = params.get("username", "")
return ConfluenceClient(base_url=base_url, token=token, username=username)
def _collect_page_ids(
self,
client: ConfluenceClient,
config: SourceConfig,
errors: list[SourceError],
) -> list[str]:
"""Sammelt alle zu verarbeitenden Seiten-IDs.
Kombiniert IDs aus pages, tree_roots und spaces.
Args:
client: Der Confluence-Client.
config: Die Quellkonfiguration.
errors: Fehlerliste zum Ergänzen.
Returns:
Deduplizierte Liste von Seiten-IDs.
"""
params = config.params
page_ids: list[str] = []
seen: set[str] = set()
def _add_ids(ids: list[str]) -> None:
for pid in ids:
if pid not in seen:
seen.add(pid)
page_ids.append(pid)
# 1. Einzelne Seiten
pages = params.get("pages", [])
if isinstance(pages, list):
_add_ids([str(p) for p in pages])
# 2. Seitenbäume
tree_roots = params.get("tree_roots", [])
if isinstance(tree_roots, list):
for root_id in tree_roots:
try:
tree_ids = client.get_page_tree(str(root_id))
_add_ids(tree_ids)
except (urllib.error.URLError, urllib.error.HTTPError) as e:
logger.warning(
"Seitenbaum %s nicht abrufbar: %s", root_id, e
)
errors.append(SourceError(
source_name=config.name,
error_type="connection",
message=f"Seitenbaum {root_id}: {e}",
timestamp=datetime.now().isoformat(),
retry=True,
))
# 3. Spaces
spaces = params.get("spaces", [])
if isinstance(spaces, list):
for space_key in spaces:
try:
space_ids = client.get_space_pages(str(space_key))
_add_ids(space_ids)
except (urllib.error.URLError, urllib.error.HTTPError) as e:
logger.warning(
"Space %s nicht abrufbar: %s", space_key, e
)
errors.append(SourceError(
source_name=config.name,
error_type="connection",
message=f"Space {space_key}: {e}",
timestamp=datetime.now().isoformat(),
retry=True,
))
return page_ids
def _page_to_artifact(self, page: ConfluencePage, context: str) -> Artifact:
"""Konvertiert eine ConfluencePage in ein Artifact.
Args:
page: Die Confluence-Seite mit Inhalt und Metadaten.
context: Der Zielkontext.
Returns:
Artifact mit Markdown-Inhalt und angereichertem Frontmatter.
"""
# HTML → Markdown
markdown_content = _html_to_markdown(page.content_html)
# Titel als Überschrift voranstellen
full_content = f"# {page.title}\n\n{markdown_content}"
# Anhänge als Links anhängen
if page.attachments:
full_content += "\n\n## Anhänge\n\n"
for att in page.attachments:
att_title = att.get("title", "Anhang")
att_url = att.get("url", "")
if att_url:
full_content += f"- [{att_title}]({att_url})\n"
else:
full_content += f"- {att_title}\n"
# Content-Hash berechnen
content_hash = f"sha256:{hashlib.sha256(full_content.encode()).hexdigest()}"
# Last-Modified parsen
created_date = date.today()
if page.last_modified:
try:
# Confluence ISO Format: 2024-01-15T10:30:00.000+01:00
dt = datetime.fromisoformat(
page.last_modified.replace("Z", "+00:00")
)
created_date = dt.date()
except (ValueError, TypeError):
pass
# Slug für Dateinamen
slug = self._slugify(page.title)
file_path = Path(f"{context}/knowledge/references/{slug}.md")
metadata = ArtifactMetadata(
type="reference",
title=page.title,
tags=[f"space:{page.space_key}"],
source_context=context,
created=created_date,
updated=created_date,
shareable=True,
content_hash=content_hash,
source={
"type": "confluence",
"url": page.url,
"space": page.space_key,
"page_id": page.page_id,
"version": str(page.version),
},
category="references",
)
return Artifact(
metadata=metadata,
content=full_content,
file_path=file_path,
)
@staticmethod
def _slugify(text: str) -> str:
"""Erzeugt einen URL-freundlichen Slug aus einem Text.
Args:
text: Eingabetext (z.B. Seitentitel).
Returns:
Slug in lowercase mit Bindestrichen.
"""
slug = text.lower().strip()
# Umlaute ersetzen
replacements = {
"ä": "ae", "ö": "oe", "ü": "ue", "ß": "ss",
"Ä": "ae", "Ö": "oe", "Ü": "ue",
}
for char, replacement in replacements.items():
slug = slug.replace(char, replacement)
# Nicht-alphanumerische Zeichen durch Bindestriche ersetzen
slug = re.sub(r"[^a-z0-9]+", "-", slug)
# Führende/nachstehende Bindestriche entfernen
slug = slug.strip("-")
return slug or "untitled"
@@ -0,0 +1,433 @@
"""Email-Quellstrategie für die ETL-Pipeline.
Verarbeitet exportierte E-Mail-Dateien (.eml, .msg) und extrahiert
Absender, Empfänger, Betreff, Datum und Body als strukturierte
Markdown-Artefakte. Anhänge werden als separate Artefakte referenziert.
Requirements: 7.1, 7.2, 7.5, 7.6
"""
from __future__ import annotations
import email
import email.utils
import hashlib
import logging
from dataclasses import dataclass
from datetime import date, datetime
from email.message import Message
from pathlib import Path
from typing import Any
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceConfig,
SourceError,
SourceStrategy,
)
logger = logging.getLogger(__name__)
__all__ = ["EmailSource"]
# Supported email file extensions
_EMAIL_EXTENSIONS = {".eml", ".msg"}
# ---------------------------------------------------------------------------
# Helper functions
# ---------------------------------------------------------------------------
def _slugify(text: str) -> str:
"""Creates a URL-friendly slug from text."""
import re
slug = text.lower().strip()
slug = re.sub(r"[^a-z0-9äöüß\s-]", "", slug)
slug = re.sub(r"[\s_]+", "-", slug)
slug = re.sub(r"-+", "-", slug)
slug = slug.strip("-")
return slug[:80] if slug else "untitled"
def _compute_content_hash(content: str) -> str:
"""Computes SHA-256 hash of content."""
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
return f"sha256:{digest}"
def _parse_email_date(date_str: str | None) -> date | None:
"""Parses an email date header into a date object."""
if not date_str:
return None
try:
parsed = email.utils.parsedate_to_datetime(date_str)
return parsed.date()
except (ValueError, TypeError):
return None
def _extract_body(msg: Message) -> str:
"""Extracts the body from an email message.
Prefers plain text, falls back to HTML with basic stripping.
"""
if msg.is_multipart():
plain_parts: list[str] = []
html_parts: list[str] = []
for part in msg.walk():
content_type = part.get_content_type()
disposition = str(part.get("Content-Disposition", ""))
# Skip attachments
if "attachment" in disposition:
continue
try:
payload = part.get_payload(decode=True)
if payload is None:
continue
charset = part.get_content_charset() or "utf-8"
text = payload.decode(charset, errors="replace")
except (LookupError, UnicodeDecodeError):
continue
if content_type == "text/plain":
plain_parts.append(text)
elif content_type == "text/html":
html_parts.append(text)
if plain_parts:
return "\n".join(plain_parts)
if html_parts:
return _strip_html("\n".join(html_parts))
return ""
else:
# Not multipart
try:
payload = msg.get_payload(decode=True)
if payload is None:
return ""
charset = msg.get_content_charset() or "utf-8"
text = payload.decode(charset, errors="replace")
except (LookupError, UnicodeDecodeError):
return ""
if msg.get_content_type() == "text/html":
return _strip_html(text)
return text
def _strip_html(html: str) -> str:
"""Basic HTML tag stripping for fallback body extraction."""
import re
# Remove script/style blocks
text = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
# Replace <br> and <p> with newlines
text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
text = re.sub(r"</?p[^>]*>", "\n", text, flags=re.IGNORECASE)
# Remove all remaining tags
text = re.sub(r"<[^>]+>", "", text)
# Collapse whitespace
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def _extract_attachments(msg: Message) -> list[dict[str, str]]:
"""Extracts attachment metadata from an email message.
Returns a list of dicts with 'filename' and 'content_type'.
"""
attachments: list[dict[str, str]] = []
if not msg.is_multipart():
return attachments
for part in msg.walk():
disposition = str(part.get("Content-Disposition", ""))
if "attachment" in disposition:
filename = part.get_filename() or "unnamed-attachment"
content_type = part.get_content_type() or "application/octet-stream"
attachments.append({
"filename": filename,
"content_type": content_type,
})
return attachments
def _format_recipients(recipients_str: str | None) -> list[str]:
"""Parses a recipient header into a list of email addresses/names."""
if not recipients_str:
return []
# Parse multiple addresses
addresses = email.utils.getaddresses([recipients_str])
result: list[str] = []
for name, addr in addresses:
if name and addr:
result.append(f"{name} <{addr}>")
elif addr:
result.append(addr)
elif name:
result.append(name)
return result
# ---------------------------------------------------------------------------
# EmailSource implementiert SourceStrategy ABC
# ---------------------------------------------------------------------------
class EmailSource(SourceStrategy):
"""Quellstrategie: Verarbeitet E-Mail-Dateien (.eml/.msg) aus einem Verzeichnis.
Extrahiert Absender, Empfänger, Betreff, Datum und Body.
Erstellt Markdown-Artefakte mit entsprechendem Frontmatter.
Anhänge werden als Links im Artefakt-Body referenziert.
.msg-Dateien erfordern das optionale Paket `extract_msg`. Ist es nicht
installiert, wird ein Fehler geloggt und die Datei übersprungen.
"""
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
"""Extrahiert E-Mail-Artefakte aus dem konfigurierten Verzeichnis.
Args:
config: Quellkonfiguration mit params["directory"] als Pfad.
context: Zielkontext (bahn, dhive, privat).
Returns:
ExtractionResult mit extrahierten Artefakten und Fehlern.
"""
directory = self._resolve_directory(config)
artifacts: list[Artifact] = []
errors: list[SourceError] = []
if not directory.exists():
errors.append(SourceError(
source_name=config.name,
error_type="parse",
message=f"Verzeichnis existiert nicht: {directory}",
timestamp="",
retry=True,
))
return ExtractionResult(artifacts=artifacts, errors=errors)
if not directory.is_dir():
errors.append(SourceError(
source_name=config.name,
error_type="parse",
message=f"Pfad ist kein Verzeichnis: {directory}",
timestamp="",
retry=False,
))
return ExtractionResult(artifacts=artifacts, errors=errors)
# Find all email files
email_files = sorted(
f for f in directory.rglob("*")
if f.is_file() and f.suffix.lower() in _EMAIL_EXTENSIONS
)
for file_path in email_files:
try:
artifact = self._process_email_file(file_path, config.name)
if artifact is not None:
artifacts.append(artifact)
except Exception as e:
logger.warning("Fehler beim Verarbeiten von %s: %s", file_path, e)
errors.append(SourceError(
source_name=config.name,
error_type="parse",
message=f"Fehler bei {file_path.name}: {e}",
timestamp="",
retry=False,
artifact_path=str(file_path),
))
return ExtractionResult(artifacts=artifacts, errors=errors)
def supports_incremental(self) -> bool:
"""Email-Quellen unterstützen keine inkrementelle Verarbeitung."""
return False
# ------------------------------------------------------------------
# Internal methods
# ------------------------------------------------------------------
def _resolve_directory(self, config: SourceConfig) -> Path:
"""Resolves the source directory from config."""
dir_param = config.params.get("directory", "")
if dir_param:
return Path(dir_param)
raise ValueError(
"EmailSource benötigt params['directory'] in der Konfiguration."
)
def _process_email_file(self, file_path: Path, source_name: str) -> Artifact | None:
"""Processes a single email file and returns an Artifact."""
suffix = file_path.suffix.lower()
if suffix == ".eml":
return self._process_eml(file_path, source_name)
elif suffix == ".msg":
return self._process_msg(file_path, source_name)
return None
def _process_eml(self, file_path: Path, source_name: str) -> Artifact:
"""Processes a .eml file using Python's built-in email module."""
raw_bytes = file_path.read_bytes()
msg = email.message_from_bytes(raw_bytes)
# Extract fields
sender = msg.get("From", "")
to_raw = msg.get("To", "")
cc_raw = msg.get("Cc", "")
subject = msg.get("Subject", "") or "Kein Betreff"
date_str = msg.get("Date", "")
message_id = msg.get("Message-ID", "")
in_reply_to = msg.get("In-Reply-To", "")
recipients = _format_recipients(to_raw)
cc_recipients = _format_recipients(cc_raw)
email_date = _parse_email_date(date_str)
body = _extract_body(msg)
attachments = _extract_attachments(msg)
# Build artifact
return self._build_artifact(
file_path=file_path,
sender=sender,
recipients=recipients,
cc_recipients=cc_recipients,
subject=subject,
email_date=email_date,
body=body,
attachments=attachments,
message_id=message_id,
in_reply_to=in_reply_to,
source_name=source_name,
)
def _process_msg(self, file_path: Path, source_name: str) -> Artifact:
"""Processes a .msg file. Requires extract_msg package."""
try:
import extract_msg # type: ignore[import-untyped]
except ImportError:
raise ImportError(
f"Das Paket 'extract_msg' wird für .msg-Dateien benötigt. "
f"Installation: pip install extract-msg"
)
msg = extract_msg.Message(str(file_path))
try:
sender = msg.sender or ""
recipients = [r.strip() for r in (msg.to or "").split(";") if r.strip()]
cc_recipients = [r.strip() for r in (msg.cc or "").split(";") if r.strip()]
subject = msg.subject or "Kein Betreff"
body = msg.body or ""
email_date = msg.date.date() if msg.date else None
message_id = msg.messageId or ""
attachments = [
{"filename": att.longFilename or att.shortFilename or "unnamed",
"content_type": att.mimetype or "application/octet-stream"}
for att in (msg.attachments or [])
]
finally:
msg.close()
return self._build_artifact(
file_path=file_path,
sender=sender,
recipients=recipients,
cc_recipients=cc_recipients,
subject=subject,
email_date=email_date,
body=body,
attachments=attachments,
message_id=message_id,
in_reply_to="",
source_name=source_name,
)
def _build_artifact(
self,
file_path: Path,
sender: str,
recipients: list[str],
cc_recipients: list[str],
subject: str,
email_date: date | None,
body: str,
attachments: list[dict[str, str]],
message_id: str,
in_reply_to: str,
source_name: str,
) -> Artifact:
"""Builds a Markdown artifact from parsed email data."""
# Build Markdown content
content_lines: list[str] = []
content_lines.append(f"# {subject}\n")
content_lines.append("")
content_lines.append(f"**Von:** {sender} ")
content_lines.append(f"**An:** {', '.join(recipients)} ")
if cc_recipients:
content_lines.append(f"**CC:** {', '.join(cc_recipients)} ")
if email_date:
content_lines.append(f"**Datum:** {email_date.isoformat()} ")
content_lines.append("")
content_lines.append("---\n")
content_lines.append("")
content_lines.append(body.strip())
# Attachments section
if attachments:
content_lines.append("")
content_lines.append("")
content_lines.append("## Anhänge\n")
content_lines.append("")
for att in attachments:
filename = att["filename"]
content_lines.append(f"- [{filename}](attachments/{filename})")
content = "\n".join(content_lines)
# Build source dict for frontmatter
source_dict: dict[str, str] = {
"type": "email",
"file": str(file_path),
}
if message_id:
source_dict["message_id"] = message_id
# Build metadata
metadata = ArtifactMetadata(
type="email",
title=subject,
tags=["email"],
source_context="",
created=email_date,
updated=email_date,
shareable=False,
source=source_dict,
category="inbox",
people=[sender] + recipients,
content_hash=_compute_content_hash(content),
)
# Generate a filename from subject
date_prefix = email_date.isoformat() if email_date else "undated"
slug = _slugify(subject)
filename = f"{date_prefix}-{slug}.md"
return Artifact(
metadata=metadata,
content=content,
file_path=Path(filename),
)
@@ -0,0 +1,483 @@
"""Jira-Quellstrategie für die ETL-Pipeline.
Extrahiert Issues via JQL-Filter aus der Jira REST API und liefert
sie als strukturierte Markdown-Artefakte zurück.
Unterstützt inkrementelle Updates via das `updated`-Feld der Jira-Issues.
Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.7
"""
from __future__ import annotations
import hashlib
import logging
from datetime import date, datetime
from typing import Any
import requests
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceConfig,
SourceError,
SourceStrategy,
)
logger = logging.getLogger(__name__)
__all__ = ["JiraSource"]
# ---------------------------------------------------------------------------
# Hilfsfunktionen
# ---------------------------------------------------------------------------
def _compute_content_hash(text: str) -> str:
"""Berechnet SHA-256-Hash eines Textes."""
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
return f"sha256:{digest}"
def _parse_jira_date(date_str: str | None) -> date | None:
"""Parsed ein Jira-Datumsfeld (ISO-8601) in ein date-Objekt.
Jira liefert Timestamps wie '2025-07-01T10:30:00.000+0200'.
"""
if not date_str:
return None
try:
# Versuche volles Datetime-Format
dt = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
return dt.date()
except (ValueError, TypeError):
pass
try:
return date.fromisoformat(date_str[:10])
except (ValueError, TypeError):
return None
def _slugify(text: str) -> str:
"""Erzeugt einen URL-freundlichen Slug aus einem Text."""
import re
slug = text.lower().strip()
slug = re.sub(r"[^a-z0-9\-]+", "-", slug)
slug = re.sub(r"-+", "-", slug)
return slug.strip("-")
def _build_issue_markdown(issue: dict[str, Any]) -> str:
"""Stellt ein Jira-Issue als Markdown-Body zusammen.
Enthält: Summary als H1, Description, Kommentare als Sub-Sections.
"""
fields = issue.get("fields", {})
summary = fields.get("summary", "Untitled Issue")
description = fields.get("description") or ""
status = fields.get("status", {}).get("name", "Unknown")
parts: list[str] = []
# Title
parts.append(f"# {summary}\n")
# Status-Info
parts.append(f"**Status:** {status}\n")
# Description
if description:
parts.append("## Description\n")
parts.append(f"{description}\n")
# Kommentare
comment_data = fields.get("comment", {})
comments = comment_data.get("comments", []) if isinstance(comment_data, dict) else []
if comments:
parts.append("## Comments\n")
for comment in comments:
author = comment.get("author", {}).get("displayName", "Unknown")
created = comment.get("created", "")[:10]
body = comment.get("body", "")
parts.append(f"### {author} ({created})\n")
parts.append(f"{body}\n")
return "\n".join(parts)
def _build_frontmatter(issue: dict[str, Any], base_url: str) -> dict[str, str]:
"""Erstellt die Frontmatter-Source-Felder für ein Jira-Issue."""
fields = issue.get("fields", {})
key = issue.get("key", "")
project = fields.get("project", {}).get("key", "")
status = fields.get("status", {}).get("name", "")
assignee = fields.get("assignee", {})
reporter = fields.get("reporter", {})
assignee_name = assignee.get("displayName", "") if assignee else ""
reporter_name = reporter.get("displayName", "") if reporter else ""
# Construct browse URL
jira_url = f"{base_url.rstrip('/')}/browse/{key}" if base_url else ""
return {
"type": "jira",
"issue_key": key,
"project": project,
"status": status,
"assignee": assignee_name,
"reporter": reporter_name,
"url": jira_url,
}
# ---------------------------------------------------------------------------
# JiraSource implementiert SourceStrategy ABC
# ---------------------------------------------------------------------------
class JiraSource(SourceStrategy):
"""Quellstrategie: Jira-Issues via JQL-Filter extrahieren.
Verwendet die Jira REST API (v2) um Issues basierend auf einem
konfigurierbaren JQL-Filter abzurufen und als Markdown-Artefakte
zurückzugeben.
Unterstützt inkrementelle Verarbeitung: Beim wiederholten Aufruf
können über `_last_updated` bereits bekannte Update-Zeitstempel
übergeben werden, um nur geänderte Issues zu verarbeiten.
SourceConfig params:
base_url: Jira-Instanz-URL (z.B. "https://jira.bahn.de")
api_key / token: API-Token für Authentifizierung
username: Benutzername für Basic Auth (optional)
jql: JQL-Filter-String
"""
def __init__(self) -> None:
"""Initialisiert die JiraSource."""
self._last_updated: dict[str, str] = {}
self._session: requests.Session | None = None
# ------------------------------------------------------------------
# SourceStrategy ABC Interface
# ------------------------------------------------------------------
def extract(self, config: SourceConfig, context: str = "") -> ExtractionResult:
"""Extrahiert Jira-Issues basierend auf der Quellkonfiguration.
Args:
config: SourceConfig mit Jira-Verbindungsparametern.
context: Der Zielkontext (bahn, dhive, privat).
Returns:
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
"""
params = config.params
base_url = params.get("base_url", "")
token = params.get("api_key") or params.get("token", "")
username = params.get("username", "")
jql = params.get("jql", "")
if not base_url:
return ExtractionResult(
errors=[
SourceError(
source_name=config.name,
error_type="config",
message="Missing required param 'base_url'",
timestamp="",
retry=False,
)
]
)
if not jql:
return ExtractionResult(
errors=[
SourceError(
source_name=config.name,
error_type="config",
message="Missing required param 'jql'",
timestamp="",
retry=False,
)
]
)
# Session aufbauen
session = self._create_session(token, username)
# Issues abrufen
try:
issues = self._fetch_issues(session, base_url, jql)
except JiraAPIError as e:
return ExtractionResult(
errors=[
SourceError(
source_name=config.name,
error_type=e.error_type,
message=str(e),
timestamp="",
retry=e.retry,
)
]
)
# Issues zu Artefakten konvertieren
artifacts: list[Artifact] = []
errors: list[SourceError] = []
skipped = 0
for issue in issues:
key = issue.get("key", "unknown")
fields = issue.get("fields", {})
updated_str = fields.get("updated", "")
# Inkrementelle Verarbeitung: Skip wenn nicht geändert
if self._last_updated.get(key) == updated_str and updated_str:
skipped += 1
continue
try:
artifact = self._issue_to_artifact(issue, base_url, context)
artifacts.append(artifact)
# Update-Timestamp merken
if updated_str:
self._last_updated[key] = updated_str
except Exception as e:
logger.warning("Fehler beim Verarbeiten von %s: %s", key, e)
errors.append(
SourceError(
source_name=config.name,
error_type="parse",
message=f"Error processing issue {key}: {e}",
timestamp="",
retry=False,
artifact_path=key,
)
)
return ExtractionResult(
artifacts=artifacts,
errors=errors,
skipped=skipped,
)
def supports_incremental(self) -> bool:
"""Jira unterstützt inkrementelle Verarbeitung via updated-Feld.
Returns:
True inkrementelle Verarbeitung wird unterstützt.
"""
return True
# ------------------------------------------------------------------
# Inkrementelle Verarbeitung
# ------------------------------------------------------------------
def set_last_updated(self, timestamps: dict[str, str]) -> None:
"""Setzt die bekannten Update-Timestamps für inkrementelle Verarbeitung.
Args:
timestamps: Mapping von Issue-Key zu letztem `updated`-Timestamp.
"""
self._last_updated = dict(timestamps)
# ------------------------------------------------------------------
# Interne Implementierung
# ------------------------------------------------------------------
def _create_session(self, token: str, username: str) -> requests.Session:
"""Erstellt eine konfigurierte HTTP-Session für Jira.
Args:
token: API-Token oder Passwort.
username: Benutzername (für Basic Auth).
Returns:
Konfigurierte requests.Session.
"""
session = requests.Session()
session.headers["Accept"] = "application/json"
session.headers["Content-Type"] = "application/json"
if username and token:
# Basic Auth (Jira Server/DC)
session.auth = (username, token)
elif token:
# Bearer Token (Jira Cloud)
session.headers["Authorization"] = f"Bearer {token}"
return session
def _fetch_issues(
self,
session: requests.Session,
base_url: str,
jql: str,
max_results: int = 100,
) -> list[dict[str, Any]]:
"""Ruft Issues aus der Jira REST API ab.
Paginiert automatisch bei großen Ergebnismengen.
Args:
session: Die HTTP-Session.
base_url: Jira-Instanz-URL.
jql: JQL-Filter.
max_results: Maximale Anzahl pro Request.
Returns:
Liste von Issue-Dictionaries.
Raises:
JiraAPIError: Bei API-Fehlern.
"""
url = f"{base_url.rstrip('/')}/rest/api/2/search"
all_issues: list[dict[str, Any]] = []
start_at = 0
while True:
params = {
"jql": jql,
"startAt": start_at,
"maxResults": max_results,
"fields": "summary,description,status,assignee,reporter,"
"project,comment,updated,created",
}
try:
response = session.get(url, params=params, timeout=30)
except requests.ConnectionError as e:
raise JiraAPIError(
f"Connection failed to {base_url}: {e}",
error_type="connection",
retry=True,
)
except requests.Timeout as e:
raise JiraAPIError(
f"Request timeout to {base_url}: {e}",
error_type="connection",
retry=True,
)
except requests.RequestException as e:
raise JiraAPIError(
f"Request error to {base_url}: {e}",
error_type="connection",
retry=True,
)
if response.status_code == 401:
raise JiraAPIError(
f"Authentication failed for {base_url}",
error_type="auth",
retry=False,
)
elif response.status_code == 403:
raise JiraAPIError(
f"Access forbidden for {base_url}",
error_type="auth",
retry=False,
)
elif response.status_code >= 400:
raise JiraAPIError(
f"API error {response.status_code} from {base_url}: "
f"{response.text[:200]}",
error_type="connection",
retry=response.status_code >= 500,
)
data = response.json()
issues = data.get("issues", [])
all_issues.extend(issues)
# Pagination
total = data.get("total", 0)
start_at += len(issues)
if start_at >= total or not issues:
break
return all_issues
def _issue_to_artifact(
self, issue: dict[str, Any], base_url: str, context: str
) -> Artifact:
"""Konvertiert ein Jira-Issue-Dict in ein Artifact.
Args:
issue: Das Issue-Dictionary von der Jira API.
base_url: Jira-Instanz-URL für Link-Generierung.
context: Der Zielkontext.
Returns:
Ein Artifact mit Markdown-Inhalt und angereichertem Frontmatter.
"""
fields = issue.get("fields", {})
key = issue.get("key", "unknown")
summary = fields.get("summary", "Untitled")
# Markdown-Body zusammenstellen
content = _build_issue_markdown(issue)
# Content-Hash
content_hash = _compute_content_hash(content)
# Source-Frontmatter
source_dict = _build_frontmatter(issue, base_url)
# Daten parsen
created = _parse_jira_date(fields.get("created"))
updated = _parse_jira_date(fields.get("updated"))
# Tags aus Projekt und Status ableiten
project_key = fields.get("project", {}).get("key", "")
status_name = fields.get("status", {}).get("name", "")
tags: list[str] = []
if project_key:
tags.append(project_key.lower())
if status_name:
tags.append(_slugify(status_name))
# Metadata zusammenbauen
metadata = ArtifactMetadata(
type="jira",
title=summary,
tags=tags,
source_context=context,
created=created,
updated=updated,
shareable=False,
content_hash=content_hash,
source=source_dict,
category="project",
)
# Dateiname aus Issue-Key ableiten
from pathlib import Path
filename = f"{key.lower()}.md"
return Artifact(
metadata=metadata,
content=content,
file_path=Path(filename),
)
# ---------------------------------------------------------------------------
# Fehlerklasse
# ---------------------------------------------------------------------------
class JiraAPIError(Exception):
"""Fehler bei der Kommunikation mit der Jira REST API."""
def __init__(self, message: str, error_type: str = "connection", retry: bool = True):
super().__init__(message)
self.error_type = error_type
self.retry = retry
@@ -0,0 +1,426 @@
"""Link-Source und LinkRegistry für Web-Link-Management.
Verwaltet Web-Links als Wissensartefakte vom Typ `link` im `links/`-Unterordner
des jeweiligen Kontext-Knowledge-Folders. Unterstützt URL-Deduplication mit
Tag-Merging, Metadaten-Fetch und Suche über URL, Titel und Tags.
Requirements: 8.1, 8.2, 8.4, 8.5, 8.6, 8.7
"""
from __future__ import annotations
import logging
import re
from datetime import date
from pathlib import Path
from urllib.parse import urlparse
import yaml
from monorepo.knowledge.artifact import (
Artifact,
ArtifactMetadata,
parse_frontmatter,
)
logger = logging.getLogger(__name__)
class LinkRegistry:
"""Manages web links as knowledge artifacts.
Links are stored as Markdown files with YAML-Frontmatter in
`{context_path}/links/`. Each link artifact has type: link and contains
url, title, tags, created in its frontmatter.
Supports:
- URL deduplication with tag merging (Req 8.7)
- Metadata fetching from target URL (Req 8.4)
- Search over URL, title, and tags (Req 8.6)
- fetch_failed flag when URL is unreachable (Req 8.5)
"""
def __init__(self, context_path: Path) -> None:
"""Initialize LinkRegistry with a context knowledge path.
Args:
context_path: Path to the context's knowledge folder,
e.g. `{monorepo_root}/bahn/knowledge/`.
"""
self.context_path = context_path
self.links_dir = context_path / "links"
def save_link(
self,
url: str,
title: str | None = None,
tags: list[str] | None = None,
) -> Path:
"""Save or update a link artifact. Deduplicates by URL.
If a link with the same URL already exists, merges tags (union of
existing + new tags) instead of creating a duplicate.
Args:
url: The web URL to save.
title: Optional title override. If None, fetched from URL.
tags: Optional list of tags.
Returns:
Path of the created or updated link artifact.
"""
if tags is None:
tags = []
# Check for existing link with same URL
existing_path = self._find_existing(url)
if existing_path is not None:
return self._merge_tags(existing_path, tags)
# Fetch metadata if title not provided
fetched_title = ""
fetched_description = ""
fetch_failed = False
if title is None:
fetched_title, fetched_description = self._fetch_metadata(url)
if fetched_title:
title = fetched_title
else:
# Use domain + path as fallback title
title = self._title_from_url(url)
fetch_failed = True
else:
# Still try to fetch for description, but don't override title
try:
_, fetched_description = self._fetch_metadata(url)
except Exception:
fetch_failed = True
# Generate filename from title
slug = self._slugify(title)
if not slug:
slug = self._slugify(self._title_from_url(url))
if not slug:
slug = "untitled-link"
filename = f"{slug}.md"
# Ensure links directory exists
self.links_dir.mkdir(parents=True, exist_ok=True)
# Avoid filename collisions
target_path = self.links_dir / filename
counter = 1
while target_path.exists():
target_path = self.links_dir / f"{slug}-{counter}.md"
counter += 1
# Build the artifact content
content = self._build_link_content(
url=url,
title=title,
tags=tags,
description=fetched_description,
fetch_failed=fetch_failed,
)
target_path.write_text(content, encoding="utf-8")
return target_path
def search_links(self, query: str) -> list[Artifact]:
"""Search links by URL, title, or tags.
Performs case-insensitive substring matching against the URL,
title, and all tags of each link artifact.
Args:
query: Search string to match against link metadata.
Returns:
List of matching Artifact objects.
"""
if not self.links_dir.exists():
return []
query_lower = query.lower()
results: list[Artifact] = []
for md_file in sorted(self.links_dir.glob("*.md")):
try:
artifact = parse_frontmatter(md_file)
except (ValueError, OSError) as e:
logger.warning("Skipping unparseable link file %s: %s", md_file, e)
continue
# Only consider link-type artifacts
if artifact.metadata.type != "link":
continue
# Match against URL
url = artifact.metadata.source.get("url", "")
if query_lower in url.lower():
results.append(artifact)
continue
# Match against title
if query_lower in artifact.metadata.title.lower():
results.append(artifact)
continue
# Match against tags
if any(query_lower in tag.lower() for tag in artifact.metadata.tags):
results.append(artifact)
continue
return results
def _fetch_metadata(self, url: str) -> tuple[str, str]:
"""Fetch page title and description from URL.
Makes an HTTP GET request to the URL and tries to extract
the <title> tag and meta description.
Args:
url: The target URL to fetch.
Returns:
Tuple of (title, description). Empty strings if fetch fails.
"""
try:
import urllib.request
import urllib.error
req = urllib.request.Request(
url,
headers={"User-Agent": "MonorepoCLI/1.0 LinkRegistry"},
)
with urllib.request.urlopen(req, timeout=10) as response:
# Read limited content (first 64KB) to avoid huge downloads
raw = response.read(65536)
charset = response.headers.get_content_charset() or "utf-8"
try:
html = raw.decode(charset, errors="replace")
except (LookupError, UnicodeDecodeError):
html = raw.decode("utf-8", errors="replace")
title = self._extract_html_title(html)
description = self._extract_meta_description(html)
return title, description
except Exception as e:
logger.debug("Failed to fetch metadata from %s: %s", url, e)
return "", ""
def _find_existing(self, url: str) -> Path | None:
"""Check for duplicate URL in context links folder.
Scans all Markdown files in the links directory and checks
if any has a matching URL in its frontmatter source.url field.
Args:
url: The URL to check for duplicates.
Returns:
Path to the existing artifact, or None if not found.
"""
if not self.links_dir.exists():
return None
for md_file in self.links_dir.glob("*.md"):
try:
artifact = parse_frontmatter(md_file)
except (ValueError, OSError):
continue
if artifact.metadata.type != "link":
continue
existing_url = artifact.metadata.source.get("url", "")
if existing_url == url:
return md_file
return None
# ------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------
def _merge_tags(self, existing_path: Path, new_tags: list[str]) -> Path:
"""Merge new tags into an existing link artifact.
Reads the existing artifact, computes union of tags,
and rewrites the file if tags changed.
Args:
existing_path: Path to the existing link artifact.
new_tags: New tags to merge in.
Returns:
Path to the updated artifact.
"""
artifact = parse_frontmatter(existing_path)
# Compute tag union (preserving order, existing first)
existing_tags = list(artifact.metadata.tags)
merged = list(existing_tags)
for tag in new_tags:
if tag not in merged:
merged.append(tag)
if merged == existing_tags:
# No change needed
return existing_path
# Rewrite with updated tags
artifact.metadata.tags = merged
content = self._render_artifact(artifact)
existing_path.write_text(content, encoding="utf-8")
return existing_path
def _render_artifact(self, artifact: Artifact) -> str:
"""Render an artifact back to its file content (frontmatter + body).
Args:
artifact: The artifact to render.
Returns:
Complete file content as string.
"""
from monorepo.knowledge.artifact import _generate_frontmatter
frontmatter = _generate_frontmatter(artifact.metadata)
return frontmatter + artifact.content
def _build_link_content(
self,
url: str,
title: str,
tags: list[str],
description: str,
fetch_failed: bool,
) -> str:
"""Build the complete Markdown content for a link artifact.
Args:
url: The link URL.
title: The link title.
tags: List of tags.
description: Fetched description (may be empty).
fetch_failed: Whether metadata fetch failed.
Returns:
Complete Markdown file content with frontmatter.
"""
today = date.today().isoformat()
# Build frontmatter data
data: dict = {
"type": "link",
"title": title,
"tags": tags,
"created": today,
"source": {"type": "link", "url": url},
"source_context": "",
"shareable": False,
}
if fetch_failed:
data["fetch_failed"] = True
# Serialize frontmatter
yaml_str = yaml.dump(
data,
default_flow_style=False,
allow_unicode=True,
sort_keys=False,
)
frontmatter = f"---\n{yaml_str}---\n"
# Build body
body_lines = [f"\n[{title}]({url})\n"]
if description:
body_lines.append(f"\n{description}\n")
body = "".join(body_lines)
return frontmatter + body
def _title_from_url(self, url: str) -> str:
"""Derive a title from a URL (domain + path).
Args:
url: The URL to derive title from.
Returns:
A human-readable title string.
"""
parsed = urlparse(url)
path_part = parsed.path.strip("/")
if path_part:
return f"{parsed.netloc}/{path_part}"
return parsed.netloc or url
def _slugify(self, text: str) -> str:
"""Convert text to URL-safe slug.
Lowercase, replace non-alphanumeric with hyphens,
collapse multiple hyphens, limit to 60 chars.
Args:
text: The text to slugify.
Returns:
A URL-safe slug string.
"""
slug = text.lower()
slug = re.sub(r"[^a-z0-9]+", "-", slug)
slug = re.sub(r"-{2,}", "-", slug)
slug = slug.strip("-")
return slug[:60]
def _extract_html_title(self, html: str) -> str:
"""Extract <title> content from HTML.
Args:
html: The HTML content string.
Returns:
The title text, or empty string if not found.
"""
match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
if match:
title = match.group(1).strip()
# Clean up HTML entities
title = re.sub(r"&amp;", "&", title)
title = re.sub(r"&lt;", "<", title)
title = re.sub(r"&gt;", ">", title)
title = re.sub(r"&quot;", '"', title)
title = re.sub(r"&#39;", "'", title)
title = re.sub(r"\s+", " ", title)
return title
return ""
def _extract_meta_description(self, html: str) -> str:
"""Extract meta description from HTML.
Args:
html: The HTML content string.
Returns:
The description text, or empty string if not found.
"""
match = re.search(
r'<meta\s+[^>]*name=["\']description["\'][^>]*content=["\']([^"\']*)["\']',
html,
re.IGNORECASE,
)
if not match:
# Try alternative ordering (content before name)
match = re.search(
r'<meta\s+[^>]*content=["\']([^"\']*)["\'][^>]*name=["\']description["\']',
html,
re.IGNORECASE,
)
if match:
return match.group(1).strip()
return ""
@@ -1,111 +1,376 @@
"""Markdown-Quellstrategie für die ETL-Pipeline.
Scannt ein Verzeichnis rekursiv nach .md-Dateien mit YAML-Frontmatter
Scannt ein Verzeichnis rekursiv nach .md/.txt-Dateien mit YAML-Frontmatter
und liefert eine Liste von Artifacts zurück.
Requirements: 3.2, 3.10, 3.12, 3.19
Implementiert das SourceStrategy-Interface (ABC) und unterstützt
inkrementelle Verarbeitung via Content-Hash.
Requirements: 2.2, 2.4, 2.7, 4.1
"""
from __future__ import annotations
import hashlib
import logging
from dataclasses import dataclass, field
from dataclasses import dataclass
from pathlib import Path
from monorepo.knowledge.artifact import Artifact, parse_frontmatter
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata, parse_frontmatter
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceConfig,
SourceError as BaseSourceError,
SourceStrategy,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Legacy-kompatibles SourceError (behält file_path/error-Interface)
# ---------------------------------------------------------------------------
@dataclass
class SourceError:
"""Fehler bei der Verarbeitung einer einzelnen Quelldatei."""
"""Fehler bei der Verarbeitung einer einzelnen Quelldatei.
Behält das alte Interface (file_path, error, retry) für Abwärtskompatibilität
mit ETLPipeline und bestehenden Tests.
"""
file_path: str
error: str
retry: bool = True
def to_base_error(self, source_name: str = "markdown") -> BaseSourceError:
"""Konvertiert in das neue BaseSourceError-Format."""
return BaseSourceError(
source_name=source_name,
error_type="parse",
message=self.error,
timestamp="",
retry=self.retry,
artifact_path=self.file_path,
)
@dataclass
class MarkdownSource:
"""Quellstrategie: Rekursives Scannen eines Verzeichnisses nach Markdown-Dateien.
Findet alle .md-Dateien im angegebenen Verzeichnis (rekursiv),
__all__ = ["MarkdownSource", "SourceError"]
# ---------------------------------------------------------------------------
# Content-Hash-Hilfsfunktion für inkrementelle Verarbeitung
# ---------------------------------------------------------------------------
def _compute_file_hash(file_path: Path) -> str:
"""Berechnet einen SHA-256-Hash des Dateiinhalts.
Wird für inkrementelle Verarbeitung verwendet nur geänderte
Dateien werden erneut verarbeitet.
Args:
file_path: Pfad zur Datei.
Returns:
Hash-String im Format "sha256:<hex>".
"""
content = file_path.read_bytes()
digest = hashlib.sha256(content).hexdigest()
return f"sha256:{digest}"
# ---------------------------------------------------------------------------
# MarkdownSource implementiert SourceStrategy ABC
# ---------------------------------------------------------------------------
# Unterstützte Dateiendungen
_SUPPORTED_EXTENSIONS = {".md", ".txt"}
class MarkdownSource(SourceStrategy):
"""Quellstrategie: Rekursives Scannen eines Verzeichnisses nach Markdown/Text-Dateien.
Findet alle .md- und .txt-Dateien im angegebenen Verzeichnis (rekursiv),
parsed jede Datei mit parse_frontmatter und liefert die resultierenden
Artifacts. Dateien ohne gültiges YAML-Frontmatter werden übersprungen
und als Warning geloggt.
Artifacts. Dateien ohne gültiges YAML-Frontmatter werden als Plain-Text-
Artefakte mit minimalem Frontmatter behandelt (.txt) oder als Fehler
geloggt (.md).
Implementiert das SourceStrategy-Interface und unterstützt zusätzlich
die alte Convenience-Schnittstelle `MarkdownSource(directory=...)`.
Inkrementelle Verarbeitung:
Bei wiederholtem Aufruf können über `_known_hashes` bereits bekannte
Datei-Hashes übergeben werden. Dateien mit unverändertem Hash werden
übersprungen (gezählt in `ExtractionResult.skipped`).
"""
directory: Path
errors: list[SourceError] = field(default_factory=list)
def __init__(self, directory: Path | None = None) -> None:
"""Initialisiert die MarkdownSource.
def extract(self) -> list[Artifact]:
"""Extrahiert alle Artefakte aus dem Quellverzeichnis.
Args:
directory: Optionaler Pfad zum Quellverzeichnis (Convenience-Interface).
Wenn angegeben, kann extract() ohne config aufgerufen werden.
"""
self._directory: Path | None = directory
self.errors: list[SourceError] = []
self._known_hashes: dict[str, str] = {}
Durchsucht das Verzeichnis rekursiv nach .md-Dateien,
parsed jede Datei und sammelt Fehler ohne abzubrechen.
# ------------------------------------------------------------------
# SourceStrategy ABC Interface
# ------------------------------------------------------------------
def extract( # type: ignore[override]
self,
config: SourceConfig | None = None,
context: str = "",
) -> ExtractionResult | list[Artifact]:
"""Extrahiert Artefakte aus dem konfigurierten Verzeichnis.
Unterstützt zwei Aufrufmodi:
1. Neues Interface: extract(config, context) → ExtractionResult
2. Legacy-Interface: extract() → list[Artifact]
Args:
config: Die Quellkonfiguration mit Verzeichnis in params["directory"].
Wenn None, wird self._directory verwendet (Legacy-Modus).
context: Der Zielkontext (bahn, dhive, privat).
Returns:
Liste erfolgreich geparster Artifacts.
ExtractionResult (neues Interface) oder list[Artifact] (Legacy).
"""
self.errors = []
artifacts: list[Artifact] = []
# Bestimme ob neues oder altes Interface verwendet wird
use_legacy = config is None
if not self.directory.exists():
logger.warning(
"Quellverzeichnis existiert nicht: %s", self.directory
# Verzeichnis auflösen
directory = self._resolve_directory(config)
# Fehler-Liste zurücksetzen
self.errors = []
# Extraktion durchführen
artifacts, errors, skipped = self._do_extract(directory)
# Fehler speichern (für Legacy-Kompatibilität)
self.errors = errors
if use_legacy:
# Legacy-Rückgabe: nur die Artefaktliste
return artifacts
else:
# Neues Interface: ExtractionResult (konvertiere lokale Fehler)
base_errors = [e.to_base_error() for e in errors]
return ExtractionResult(
artifacts=artifacts,
errors=base_errors,
skipped=skipped,
)
self.errors.append(
def supports_incremental(self) -> bool:
"""Markdown/Text-Quellen unterstützen inkrementelle Verarbeitung.
Über Content-Hashes werden nur geänderte Dateien erneut verarbeitet.
Returns:
True inkrementelle Verarbeitung wird unterstützt.
"""
return True
# ------------------------------------------------------------------
# Inkrementelle Verarbeitung
# ------------------------------------------------------------------
def set_known_hashes(self, hashes: dict[str, str]) -> None:
"""Setzt die bekannten Datei-Hashes für inkrementelle Verarbeitung.
Args:
hashes: Mapping von Dateipfad (str) zu Content-Hash.
Dateien mit unverändertem Hash werden übersprungen.
"""
self._known_hashes = dict(hashes)
# ------------------------------------------------------------------
# Interne Implementierung
# ------------------------------------------------------------------
def _resolve_directory(self, config: SourceConfig | None) -> Path:
"""Löst das Quellverzeichnis aus config oder self._directory auf.
Args:
config: Optionale SourceConfig mit params["directory"].
Returns:
Path zum Quellverzeichnis.
Raises:
ValueError: Wenn weder config noch self._directory gesetzt ist.
"""
if config is not None:
dir_param = config.params.get("directory", "")
if dir_param:
return Path(dir_param)
if self._directory is not None:
return self._directory
raise ValueError(
"Kein Verzeichnis angegeben: "
"weder config.params['directory'] noch directory-Parameter gesetzt."
)
def _do_extract(
self, directory: Path
) -> tuple[list[Artifact], list[SourceError], int]:
"""Führt die eigentliche Extraktion durch.
Args:
directory: Das zu scannende Verzeichnis.
Returns:
Tuple von (artifacts, errors, skipped_count).
"""
artifacts: list[Artifact] = []
errors: list[SourceError] = []
skipped = 0
# Verzeichnis-Validierung
if not directory.exists():
logger.warning(
"Quellverzeichnis existiert nicht: %s", directory
)
errors.append(
SourceError(
file_path=str(self.directory),
error=f"Verzeichnis existiert nicht: {self.directory}",
file_path=str(directory),
error=f"Verzeichnis existiert nicht: {directory}",
retry=True,
)
)
return artifacts
return artifacts, errors, skipped
if not self.directory.is_dir():
if not directory.is_dir():
logger.warning(
"Quellpfad ist kein Verzeichnis: %s", self.directory
"Quellpfad ist kein Verzeichnis: %s", directory
)
self.errors.append(
errors.append(
SourceError(
file_path=str(self.directory),
error=f"Pfad ist kein Verzeichnis: {self.directory}",
file_path=str(directory),
error=f"Pfad ist kein Verzeichnis: {directory}",
retry=False,
)
)
return artifacts
return artifacts, errors, skipped
md_files = sorted(self.directory.rglob("*.md"))
# Alle unterstützten Dateien finden
all_files = sorted(
f
for f in directory.rglob("*")
if f.is_file() and f.suffix.lower() in _SUPPORTED_EXTENSIONS
)
for md_file in md_files:
try:
artifact = parse_frontmatter(md_file)
for file_path in all_files:
# Inkrementelle Verarbeitung: Hash-Check
if self._known_hashes:
file_key = str(file_path)
current_hash = _compute_file_hash(file_path)
known_hash = self._known_hashes.get(file_key)
if known_hash == current_hash:
skipped += 1
continue
# Datei verarbeiten
artifact = self._extract_file(file_path, errors)
if artifact is not None:
artifacts.append(artifact)
except ValueError as e:
logger.warning(
"Überspringe Datei ohne gültiges Frontmatter: %s %s",
md_file,
e,
)
self.errors.append(
SourceError(
file_path=str(md_file),
error=str(e),
retry=False,
)
)
except OSError as e:
logger.error(
"Fehler beim Lesen der Datei: %s %s", md_file, e
)
self.errors.append(
SourceError(
file_path=str(md_file),
error=str(e),
retry=True,
)
)
return artifacts
return artifacts, errors, skipped
def _extract_file(
self, file_path: Path, errors: list[SourceError]
) -> Artifact | None:
"""Extrahiert ein einzelnes Artefakt aus einer Datei.
.md-Dateien: Parsed mit parse_frontmatter (erwartet Frontmatter).
.txt-Dateien: Liest als Plain-Text und erstellt minimales Artefakt.
Args:
file_path: Pfad zur Datei.
errors: Fehlerliste zum Ergänzen bei Problemen.
Returns:
Artifact oder None bei Fehler.
"""
try:
if file_path.suffix.lower() == ".txt":
return self._extract_txt_file(file_path)
else:
return parse_frontmatter(file_path)
except ValueError as e:
logger.warning(
"Überspringe Datei ohne gültiges Frontmatter: %s %s",
file_path,
e,
)
errors.append(
SourceError(
file_path=str(file_path),
error=str(e),
retry=False,
)
)
return None
except OSError as e:
logger.error(
"Fehler beim Lesen der Datei: %s %s", file_path, e
)
errors.append(
SourceError(
file_path=str(file_path),
error=str(e),
retry=True,
)
)
return None
def _extract_txt_file(self, file_path: Path) -> Artifact:
"""Extrahiert ein Artefakt aus einer .txt-Datei.
Erstellt ein minimales Artefakt ohne Frontmatter Titel wird
aus dem Dateinamen abgeleitet.
Args:
file_path: Pfad zur .txt-Datei.
Returns:
Artifact mit minimalem Metadata.
"""
content = file_path.read_text(encoding="utf-8")
title = file_path.stem.replace("-", " ").replace("_", " ").title()
metadata = ArtifactMetadata(
type="note",
title=title,
source={"type": "file", "file": str(file_path)},
)
return Artifact(
metadata=metadata,
content=content,
file_path=file_path,
)
# ------------------------------------------------------------------
# Legacy-Kompatibilität: Attribut-Zugriff
# ------------------------------------------------------------------
@property
def directory(self) -> Path | None:
"""Zugriff auf das konfigurierte Verzeichnis (Legacy-Kompatibilität)."""
return self._directory
@directory.setter
def directory(self, value: Path) -> None:
"""Setzt das Verzeichnis (Legacy-Kompatibilität)."""
self._directory = value
@@ -0,0 +1,407 @@
"""PDF- und OCR-Quellstrategie für die ETL-Pipeline.
Implementiert die Extraktion von Text aus PDF-Dateien (eingebetteter Text
und OCR-Fallback für gescannte Seiten) sowie die direkte OCR-Verarbeitung
von Bilddateien (PNG, JPG, JPEG).
OCR-Strategie (Priorität):
1. Kiro Vision Bilder werden an den AI-Agenten gesendet (primär)
2. Tesseract (optional, spätere Alternative für CI/Batch)
Requirements: 4.2, 4.3, 4.4, 4.5, 4.6
"""
from __future__ import annotations
import logging
from datetime import date, datetime
from pathlib import Path
from typing import Any
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata, compute_content_hash
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceConfig,
SourceError,
SourceStrategy,
)
logger = logging.getLogger(__name__)
# Unterstützte Dateierweiterungen
PDF_EXTENSIONS = {".pdf"}
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg"}
SUPPORTED_EXTENSIONS = PDF_EXTENSIONS | IMAGE_EXTENSIONS
# OCR-Sprachkonfiguration
OCR_LANGUAGES = "deu+eng"
# ---------------------------------------------------------------------------
# PDF-Textextraktion
# ---------------------------------------------------------------------------
def _extract_pdf_text(file_path: Path) -> tuple[str, bool]:
"""Extrahiert Text aus einer PDF-Datei.
Versucht eingebetteten Text via pypdf zu extrahieren. Falls Seiten
keinen Text enthalten (gescannte PDFs), wird ein OCR-Fallback benötigt.
Args:
file_path: Pfad zur PDF-Datei.
Returns:
Tuple (extrahierter_text, needs_ocr).
needs_ocr ist True wenn mindestens eine Seite keinen Text enthält.
Raises:
ImportError: Wenn keine PDF-Bibliothek verfügbar ist.
OSError: Bei Dateizugriffsfehlern.
"""
try:
from pypdf import PdfReader # type: ignore[import-untyped]
except ImportError:
try:
from PyPDF2 import PdfReader # type: ignore[import-untyped, no-redef]
except ImportError:
raise ImportError(
"Keine PDF-Bibliothek verfügbar. "
"Installiere 'pypdf' oder 'PyPDF2': pip install pypdf"
)
reader = PdfReader(str(file_path))
pages_text: list[str] = []
pages_without_text: list[int] = []
for i, page in enumerate(reader.pages):
text = page.extract_text() or ""
text = text.strip()
if text:
pages_text.append(text)
else:
pages_without_text.append(i)
full_text = "\n\n".join(pages_text)
needs_ocr = len(pages_without_text) > 0
if needs_ocr:
logger.info(
"PDF %s: %d von %d Seiten ohne Text (OCR benötigt für Seiten: %s)",
file_path.name,
len(pages_without_text),
len(reader.pages),
pages_without_text,
)
return full_text, needs_ocr
# ---------------------------------------------------------------------------
# OCR-Engine (Kiro Vision als primärer Provider)
# ---------------------------------------------------------------------------
def _perform_ocr(file_path: Path, languages: str = OCR_LANGUAGES) -> tuple[str, bool]:
"""Führt OCR auf einer Bild- oder PDF-Datei durch.
Primärer Provider: Kiro Vision (über kiro_client-Integration).
Fallback: Stub, der ocr_failed=True zurückgibt wenn Kiro nicht verfügbar.
Args:
file_path: Pfad zur Bild- oder PDF-Datei.
languages: Sprachkonfiguration (default: deu+eng).
Returns:
Tuple (extrahierter_text, ocr_failed).
ocr_failed ist True wenn OCR fehlgeschlagen ist.
"""
# Versuche Kiro Vision Client
try:
from monorepo.knowledge.integrations.kiro_client import ( # type: ignore[import-untyped]
KiroClient,
)
client = KiroClient()
text = client.ocr(file_path, languages=languages)
if text and text.strip():
return text.strip(), False
else:
logger.warning(
"Kiro Vision lieferte keinen Text für: %s", file_path
)
return "", True
except ImportError:
logger.info(
"kiro_client nicht verfügbar, versuche Tesseract-Fallback für: %s",
file_path,
)
except Exception as e:
logger.warning(
"Kiro Vision OCR fehlgeschlagen für %s: %s", file_path, e
)
# Fallback: Tesseract (optional)
try:
import pytesseract # type: ignore[import-untyped]
from PIL import Image # type: ignore[import-untyped]
if file_path.suffix.lower() in IMAGE_EXTENSIONS:
img = Image.open(file_path)
text = pytesseract.image_to_string(img, lang=languages.replace("+", "+"))
if text and text.strip():
return text.strip(), False
else:
# Für PDFs: Tesseract kann nur Bilder verarbeiten
logger.info(
"Tesseract-OCR nur für Bilddateien verfügbar, nicht für PDF-Seiten: %s",
file_path,
)
return "", True
except ImportError:
logger.info(
"Weder kiro_client noch pytesseract verfügbar für OCR: %s",
file_path,
)
except Exception as e:
logger.warning("Tesseract OCR fehlgeschlagen für %s: %s", file_path, e)
return "", True
# ---------------------------------------------------------------------------
# PDFSource Strategie
# ---------------------------------------------------------------------------
class PDFSource(SourceStrategy):
"""Quellstrategie für PDF- und Bilddateien.
Verarbeitet:
- PDF-Dateien: Eingebetteten Text extrahieren, OCR-Fallback für gescannte Seiten
- Bilddateien (PNG, JPG, JPEG): Direkt an OCR weiterleiten
Setzt `ocr_failed: true` im Artefakt-Frontmatter falls OCR fehlschlägt.
Behält Referenz auf Originaldatei in `source.file`.
"""
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
"""Extrahiert Artefakte aus PDF- und Bilddateien.
Args:
config: SourceConfig mit params["path"] oder params["directory"].
context: Der Zielkontext (bahn, dhive, privat).
Returns:
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
"""
result = ExtractionResult()
# Pfad aus Konfiguration ermitteln
source_path = self._resolve_source_path(config)
if source_path is None:
result.errors.append(
SourceError(
source_name=config.name,
error_type="parse",
message="Kein gültiger Pfad in config.params['path'] oder config.params['directory']",
timestamp=datetime.now().isoformat(),
retry=False,
)
)
return result
# Dateien sammeln
files = self._collect_files(source_path)
if not files:
logger.info("Keine unterstützten Dateien gefunden in: %s", source_path)
return result
# Dateien verarbeiten
for file_path in files:
try:
artifact = self._process_file(file_path, config, context)
if artifact is not None:
result.artifacts.append(artifact)
except Exception as e:
logger.error(
"Fehler bei der Verarbeitung von %s: %s", file_path, e
)
result.errors.append(
SourceError(
source_name=config.name,
error_type="parse",
message=f"Fehler bei {file_path.name}: {e}",
timestamp=datetime.now().isoformat(),
retry=True,
artifact_path=str(file_path),
)
)
return result
def supports_incremental(self) -> bool:
"""PDF-Source unterstützt inkrementelle Verarbeitung via Content-Hash."""
return True
# ------------------------------------------------------------------
# Hilfsmethoden
# ------------------------------------------------------------------
def _resolve_source_path(self, config: SourceConfig) -> Path | None:
"""Ermittelt den Quellpfad aus der Konfiguration."""
path_str = config.params.get("path") or config.params.get("directory") or ""
if not path_str:
return None
source_path = Path(path_str)
if not source_path.exists():
return None
return source_path
def _collect_files(self, source_path: Path) -> list[Path]:
"""Sammelt alle unterstützten Dateien aus dem Quellpfad."""
if source_path.is_file():
if source_path.suffix.lower() in SUPPORTED_EXTENSIONS:
return [source_path]
return []
# Verzeichnis rekursiv durchsuchen
files: list[Path] = []
for ext in SUPPORTED_EXTENSIONS:
files.extend(source_path.rglob(f"*{ext}"))
return sorted(files)
def _process_file(
self, file_path: Path, config: SourceConfig, context: str
) -> Artifact | None:
"""Verarbeitet eine einzelne PDF- oder Bilddatei.
Args:
file_path: Pfad zur Datei.
config: Die Quellkonfiguration.
context: Der Zielkontext.
Returns:
Ein Artifact oder None bei Fehler.
"""
suffix = file_path.suffix.lower()
if suffix in PDF_EXTENSIONS:
return self._process_pdf(file_path, config, context)
elif suffix in IMAGE_EXTENSIONS:
return self._process_image(file_path, config, context)
else:
return None
def _process_pdf(
self, file_path: Path, config: SourceConfig, context: str
) -> Artifact:
"""Verarbeitet eine PDF-Datei.
1. Versucht eingebetteten Text zu extrahieren
2. Falls Seiten ohne Text: OCR-Fallback
3. Setzt ocr_failed bei Fehlschlag
"""
ocr_failed = False
text = ""
try:
embedded_text, needs_ocr = _extract_pdf_text(file_path)
text = embedded_text
if needs_ocr and not embedded_text:
# Komplett gescanntes PDF OCR für das gesamte Dokument
ocr_text, ocr_failed = _perform_ocr(file_path)
if ocr_text:
text = ocr_text
elif needs_ocr and embedded_text:
# Teilweise gescannt Text vorhanden, OCR nur für fehlende Seiten
# Für jetzt verwenden wir den vorhandenen Text
logger.info(
"PDF %s: Eingebetteter Text vorhanden, gescannte Seiten übersprungen",
file_path.name,
)
except ImportError as e:
logger.warning("PDF-Bibliothek nicht verfügbar: %s", e)
# Versuche OCR als Fallback
ocr_text, ocr_failed = _perform_ocr(file_path)
text = ocr_text
return self._create_artifact(
file_path=file_path,
text=text,
context=context,
source_type="pdf",
ocr_failed=ocr_failed,
)
def _process_image(
self, file_path: Path, config: SourceConfig, context: str
) -> Artifact:
"""Verarbeitet eine Bilddatei via OCR.
Bilddateien werden direkt an die OCR-Engine weitergeleitet.
"""
text, ocr_failed = _perform_ocr(file_path)
return self._create_artifact(
file_path=file_path,
text=text,
context=context,
source_type="image",
ocr_failed=ocr_failed,
)
def _create_artifact(
self,
file_path: Path,
text: str,
context: str,
source_type: str,
ocr_failed: bool,
) -> Artifact:
"""Erstellt ein Artifact aus den extrahierten Daten.
Args:
file_path: Pfad zur Originaldatei.
text: Der extrahierte Text.
context: Der Zielkontext.
source_type: "pdf" oder "image".
ocr_failed: Ob OCR fehlgeschlagen ist.
Returns:
Ein vollständiges Artifact.
"""
# Titel aus Dateinamen ableiten
title = file_path.stem.replace("-", " ").replace("_", " ").strip()
if not title:
title = file_path.name
# Content-Hash berechnen
content_hash = compute_content_hash(text) if text else ""
metadata = ArtifactMetadata(
type="reference",
title=title,
tags=[source_type],
source_context=context,
created=date.today(),
updated=date.today(),
shareable=False,
content_hash=content_hash,
source={
"type": source_type,
"file": str(file_path),
},
category="references",
ocr_failed=ocr_failed,
)
# Markdown-Inhalt zusammenbauen
content = f"# {title}\n\n{text}\n" if text else f"# {title}\n\n"
return Artifact(
metadata=metadata,
content=content,
file_path=file_path,
)
@@ -0,0 +1,243 @@
"""Wissensdatenbank-Quellstrategie für die ETL-Pipeline.
Adapter für `bahn/wissensdatenbank/output/` liest Markdown-Artefakte
als Read-Only-Referenzen in den bahn/knowledge/ YAML-Index ein, ohne
die Originaldateien zu verändern oder dem Google OKF-Format aufzuzwingen.
Requirements: 13.1, 13.2, 13.3, 13.4, 13.5
"""
from __future__ import annotations
import hashlib
import logging
from datetime import date
from pathlib import Path
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
from monorepo.knowledge.sources.base import (
ExtractionResult,
SourceConfig,
SourceError,
SourceStrategy,
)
logger = logging.getLogger(__name__)
__all__ = ["WissensdatenbankSource"]
# Unterstützte Dateiendungen in der Wissensdatenbank
_SUPPORTED_EXTENSIONS = {".md", ".txt"}
def _compute_file_hash(file_path: Path) -> str:
"""Berechnet den SHA-256-Hash des Dateiinhalts.
Args:
file_path: Pfad zur Datei.
Returns:
Hash-String im Format "sha256:<hex>".
"""
content = file_path.read_bytes()
digest = hashlib.sha256(content).hexdigest()
return f"sha256:{digest}"
class WissensdatenbankSource(SourceStrategy):
"""Quellstrategie: Read-Only-Referenzen aus bahn/wissensdatenbank/output/.
Scannt das Wissensdatenbank-Output-Verzeichnis nach Markdown-Dateien
und erstellt Artifact-Objekte mit `source: wissensdatenbank`-Metadaten.
**Kernverhalten:**
- Originaldateien werden NICHT modifiziert (Read-Only-Referenz)
- Kein Google-OKF-Reformat (kein Splitting, keine index.md-Generierung)
- Artefakte werden als Referenzen im Index geführt (source: wissensdatenbank)
- Inkrementelle Verarbeitung via Content-Hash
Requirements: 13.1, 13.2, 13.3, 13.4, 13.5
"""
def __init__(self) -> None:
"""Initialisiert die WissensdatenbankSource."""
self._known_hashes: dict[str, str] = {}
def extract(self, config: SourceConfig, context: str) -> ExtractionResult:
"""Extrahiert Read-Only-Referenzen aus dem Wissensdatenbank-Output.
Scannt `params["output_path"]` rekursiv nach .md/.txt-Dateien und
erstellt Artifact-Objekte ohne die Originaldateien zu verändern.
Args:
config: Die Quellkonfiguration mit `params["output_path"]`.
context: Der Zielkontext (typischerweise "bahn").
Returns:
ExtractionResult mit extrahierten Artefakten und ggf. Fehlern.
"""
output_path_str = config.params.get("output_path", "")
if not output_path_str:
return ExtractionResult(
errors=[
SourceError(
source_name=config.name,
error_type="parse",
message="Kein 'output_path' in params konfiguriert.",
timestamp="",
retry=False,
)
]
)
output_path = Path(output_path_str)
if not output_path.exists():
return ExtractionResult(
errors=[
SourceError(
source_name=config.name,
error_type="connection",
message=f"Wissensdatenbank-Verzeichnis existiert nicht: {output_path}",
timestamp="",
retry=True,
)
]
)
if not output_path.is_dir():
return ExtractionResult(
errors=[
SourceError(
source_name=config.name,
error_type="parse",
message=f"Pfad ist kein Verzeichnis: {output_path}",
timestamp="",
retry=False,
)
]
)
artifacts: list[Artifact] = []
errors: list[SourceError] = []
skipped = 0
# Alle unterstützten Dateien finden (rekursiv)
all_files = sorted(
f
for f in output_path.rglob("*")
if f.is_file() and f.suffix.lower() in _SUPPORTED_EXTENSIONS
)
for file_path in all_files:
# Inkrementelle Verarbeitung: Hash-Check
file_key = str(file_path)
try:
current_hash = _compute_file_hash(file_path)
except OSError as e:
logger.error(
"Fehler beim Lesen der Datei: %s %s", file_path, e
)
errors.append(
SourceError(
source_name=config.name,
error_type="parse",
message=f"Fehler beim Lesen: {file_path} {e}",
timestamp="",
retry=True,
artifact_path=file_key,
)
)
continue
known_hash = self._known_hashes.get(file_key)
if known_hash == current_hash:
skipped += 1
continue
# Datei lesen und als Read-Only-Referenz erstellen
try:
artifact = self._create_reference_artifact(
file_path, current_hash, context
)
artifacts.append(artifact)
except OSError as e:
logger.error(
"Fehler beim Verarbeiten: %s %s", file_path, e
)
errors.append(
SourceError(
source_name=config.name,
error_type="parse",
message=f"Fehler beim Verarbeiten: {file_path} {e}",
timestamp="",
retry=True,
artifact_path=file_key,
)
)
return ExtractionResult(
artifacts=artifacts,
errors=errors,
skipped=skipped,
)
def supports_incremental(self) -> bool:
"""Wissensdatenbank unterstützt inkrementelle Verarbeitung via Hash.
Returns:
True inkrementelle Verarbeitung wird unterstützt.
"""
return True
def set_known_hashes(self, hashes: dict[str, str]) -> None:
"""Setzt die bekannten Datei-Hashes für inkrementelle Verarbeitung.
Args:
hashes: Mapping von Dateipfad (str) zu Content-Hash.
Dateien mit unverändertem Hash werden übersprungen.
"""
self._known_hashes = dict(hashes)
# ------------------------------------------------------------------
# Interne Implementierung
# ------------------------------------------------------------------
def _create_reference_artifact(
self, file_path: Path, content_hash: str, context: str
) -> Artifact:
"""Erstellt ein Read-Only-Referenz-Artefakt aus einer Wissensdatenbank-Datei.
Die Originaldatei wird gelesen aber NICHT verändert. Das Artefakt
erhält `source: {"type": "wissensdatenbank", "path": <original_path>}`
als Metadaten.
Args:
file_path: Pfad zur Originaldatei in wissensdatenbank/output/.
content_hash: Bereits berechneter SHA-256-Hash.
context: Der Zielkontext (typischerweise "bahn").
Returns:
Artifact mit Read-Only-Referenz-Metadaten.
"""
content = file_path.read_text(encoding="utf-8")
# Titel aus Dateiname ableiten (ersetzt Bindestriche/Underscores)
title = file_path.stem.replace("-", " ").replace("_", " ").title()
metadata = ArtifactMetadata(
type="reference",
title=title,
source_context=context,
source={"type": "wissensdatenbank", "path": str(file_path)},
category="references",
content_hash=content_hash,
created=date.today(),
)
return Artifact(
metadata=metadata,
content=content,
file_path=file_path,
)
@@ -4,6 +4,8 @@ Basiert auf der DB-Wissensdatenbank-ETL-Architektur und bietet:
- Ingestion von Wissensartefakten aus verschiedenen Quellen
- Progressive-Disclosure-Index (kompakte YAML-Metadaten, Schicht 1)
- Volltextsuche mit Scope-basierter Zugriffskontrolle und Relevanz-Sortierung
- Kontextübergreifende Suche über alle Knowledge-Folder _index.yaml-Dateien
- Agent-Context-Injection: max. 10 relevante Artefakt-Pfade pro Anfrage
- Graph-Verknüpfungen zwischen Artefakten
- Timeout-Handling für Suchanfragen (5 Sekunden)
"""
@@ -26,6 +28,12 @@ logger = logging.getLogger(__name__)
# Default-Timeout für Suchanfragen in Sekunden
SEARCH_TIMEOUT_SECONDS: float = 5.0
# Max. Artefakt-Pfade für Agent-Context-Injection
MAX_CONTEXT_INJECTION_RESULTS: int = 10
# Bekannte Kontext-Knowledge-Folder (relativ zum Monorepo-Root)
CONTEXT_KNOWLEDGE_FOLDERS: list[str] = ["bahn", "dhive", "privat"]
@dataclass
class SearchResult:
@@ -48,6 +56,24 @@ class SearchResult:
partial_results: bool = False
@dataclass
class ContextInjectionResult:
"""Ergebnis der Agent-Context-Injection.
Enthält die relevantesten Artefakt-Pfade für den Agent-Prompt,
sortiert nach Relevanz-Score. Maximal MAX_CONTEXT_INJECTION_RESULTS Einträge.
Attributes:
paths: Liste der relevantesten Artefakt-Pfade (max. 10).
entries: Die zugehörigen SearchResult-Objekte (Progressive Disclosure Schicht 1).
partial_results: True wenn die Suche wegen Timeout abgebrochen wurde.
"""
paths: list[str] = field(default_factory=list)
entries: list[SearchResult] = field(default_factory=list)
partial_results: bool = False
def _compute_relevance(query_lower: str, entry: IndexEntry) -> float:
"""Berechnet den Relevanz-Score für einen Index-Eintrag.
@@ -115,6 +141,11 @@ class KnowledgeStore:
- Schicht 2 (Dokument): Vollständige Markdown-Datei mit YAML-Frontmatter
Agenten lesen zuerst den Index und laden nur bei Bedarf einzelne Dokumente.
Kontextübergreifende Suche:
- Durchsucht alle Knowledge-Folder _index.yaml-Dateien (bahn, dhive, privat)
- Respektiert ctx-guard-Sicherheitsgrenzen (nur autorisierte Scopes)
- Agent-Context-Injection: max. 10 relevante Pfade pro Anfrage
"""
INDEX_FILENAME = "_index.yaml"
@@ -124,7 +155,7 @@ class KnowledgeStore:
Args:
base_path: Basispfad des Wissensspeichers
(z.B. shared/knowledge-store/).
(z.B. shared/knowledge-store/ oder Monorepo-Root).
scope_config: Konfiguration des Scope-Mappings
(Arbeitskontext → Scope).
"""
@@ -350,3 +381,222 @@ class KnowledgeStore:
# Datei zurückschreiben (mit aktualisiertem Frontmatter)
write_artifact(artifact, self.base_path)
# -----------------------------------------------------------------------
# Kontextübergreifende Suche über Knowledge-Folder
# -----------------------------------------------------------------------
def _load_knowledge_folder_indices(self) -> list[YAMLIndex]:
"""Lädt alle _index.yaml-Dateien aus den Knowledge-Foldern.
Durchsucht die bekannten Kontext-Knowledge-Folder (bahn, dhive, privat)
und lädt deren _index.yaml als YAMLIndex-Instanzen.
Returns:
Liste geladener YAMLIndex-Instanzen (nur existierende Dateien).
"""
indices: list[YAMLIndex] = []
for context_name in CONTEXT_KNOWLEDGE_FOLDERS:
index_path = self.base_path / context_name / "knowledge" / self.INDEX_FILENAME
if index_path.exists():
try:
idx = YAMLIndex(index_path)
idx.load()
indices.append(idx)
except Exception as e:
logger.warning(
"Konnte Knowledge-Folder-Index nicht laden: %s (%s)",
index_path,
e,
)
return indices
def search_cross_context(
self,
query: str,
allowed_scopes: list[str],
timeout: float | None = None,
) -> list[SearchResult]:
"""Kontextübergreifende Suche über alle Knowledge-Folder _index.yaml-Dateien.
Durchsucht sowohl den eigenen Index als auch alle Kontext-Knowledge-Folder
(_index.yaml in bahn/knowledge/, dhive/knowledge/, privat/knowledge/).
Respektiert ctx-guard-Sicherheitsgrenzen: nur Artefakte aus autorisierten
Scopes werden zurückgegeben.
Progressive Disclosure (Schicht 1): Die Suche operiert ausschließlich
auf dem YAML-Index (kompakte Metadaten). Vollständige Dokumente werden
nur bei explizitem Nachladen über den Pfad bereitgestellt.
Args:
query: Suchbegriff für die Volltextsuche.
allowed_scopes: Liste der Scopes, aus denen Ergebnisse
geliefert werden dürfen (ctx-guard-Autorisierung).
timeout: Timeout in Sekunden (None = SEARCH_TIMEOUT_SECONDS).
Returns:
Liste von SearchResult-Objekten, sortiert nach Relevanz
(höchster Score zuerst). Enthält Ergebnisse aus dem eigenen
Index UND allen Knowledge-Folder-Indices.
Note:
Requirements: 11.1, 11.3, 11.5
"""
if not query:
return []
effective_timeout = timeout if timeout is not None else SEARCH_TIMEOUT_SECONDS
query_lower = query.lower()
results: list[SearchResult] = []
seen_ids: set[str] = set()
timed_out = False
start_time = time.monotonic()
# 1. Eigenen Index durchsuchen (shared KnowledgeStore)
for entry in self.index.entries.values():
elapsed = time.monotonic() - start_time
if elapsed >= effective_timeout:
timed_out = True
break
if entry.scope not in allowed_scopes:
continue
searchable_text = " ".join([
entry.title, entry.id, entry.summary, " ".join(entry.tags),
]).lower()
if query_lower in searchable_text:
relevance = _compute_relevance(query_lower, entry)
results.append(SearchResult(
entry=entry, relevance_score=relevance, partial_results=False,
))
seen_ids.add(entry.id)
# 2. Knowledge-Folder-Indices durchsuchen
if not timed_out:
folder_indices = self._load_knowledge_folder_indices()
for folder_index in folder_indices:
if timed_out:
break
for entry in folder_index.entries.values():
elapsed = time.monotonic() - start_time
if elapsed >= effective_timeout:
timed_out = True
break
# Scope-Filter: Nur autorisierte Scopes
if entry.scope not in allowed_scopes:
continue
# Deduplizierung: Einträge die bereits im eigenen Index sind
if entry.id in seen_ids:
continue
searchable_text = " ".join([
entry.title, entry.id, entry.summary, " ".join(entry.tags),
]).lower()
if query_lower in searchable_text:
relevance = _compute_relevance(query_lower, entry)
results.append(SearchResult(
entry=entry, relevance_score=relevance, partial_results=False,
))
seen_ids.add(entry.id)
# Bei Timeout: Alle Ergebnisse als partial markieren
if timed_out:
for result in results:
result.partial_results = True
# Nach Relevanz sortieren (höchster Score zuerst)
results.sort(key=lambda r: r.relevance_score, reverse=True)
return results
def get_context_injection(
self,
query: str,
allowed_scopes: list[str],
max_results: int | None = None,
timeout: float | None = None,
) -> ContextInjectionResult:
"""Agent-Context-Injection: Liefert die relevantesten Artefakt-Pfade.
Identifiziert die relevantesten Artefakte aus den YAML-Indices aller
autorisierten Kontexte und liefert deren Pfade als Kontextinformation
für den Agent-Prompt.
Progressive Disclosure:
- Schicht 1: YAML-Index-Daten werden für die Relevanzprüfung verwendet.
- Schicht 2: Nur die Pfade werden zurückgegeben das vollständige
Dokument wird vom Agenten bei Bedarf nachgeladen.
Args:
query: Suchbegriff oder Task-Beschreibung für Relevanzprüfung.
allowed_scopes: Autorisierte Scopes (ctx-guard-Filterung).
max_results: Maximale Anzahl zurückgegebener Pfade
(None = MAX_CONTEXT_INJECTION_RESULTS = 10).
timeout: Timeout in Sekunden (None = SEARCH_TIMEOUT_SECONDS).
Returns:
ContextInjectionResult mit max. 10 Pfaden, sortiert nach
Relevanz-Score (höchster zuerst).
Note:
Requirements: 11.2, 11.5, 11.6
"""
effective_max = max_results if max_results is not None else MAX_CONTEXT_INJECTION_RESULTS
# Kontextübergreifende Suche durchführen
all_results = self.search_cross_context(query, allowed_scopes, timeout)
# Auf max. Ergebnisse beschränken
top_results = all_results[:effective_max]
# Pfade extrahieren (nur Einträge mit gültigem Pfad)
paths: list[str] = []
filtered_results: list[SearchResult] = []
for result in top_results:
if result.entry.path:
paths.append(result.entry.path)
filtered_results.append(result)
partial = any(r.partial_results for r in top_results) if top_results else False
return ContextInjectionResult(
paths=paths,
entries=filtered_results,
partial_results=partial,
)
def load_artifact_content(self, artifact_path: str) -> str | None:
"""Progressive Disclosure Schicht 2: Lädt den vollständigen Artefakt-Inhalt.
Wird vom Agenten aufgerufen, wenn nach der Relevanzprüfung (Schicht 1)
der vollständige Dokumentinhalt benötigt wird.
Args:
artifact_path: Relativer Pfad zum Artefakt (aus dem IndexEntry.path).
Returns:
Der vollständige Dateiinhalt als String, oder None wenn die Datei
nicht existiert oder nicht gelesen werden kann.
Note:
Requirements: 11.3
"""
full_path = self.base_path / artifact_path
if not full_path.exists():
logger.debug("Artefakt-Datei nicht gefunden für Schicht-2-Zugriff: %s", full_path)
return None
try:
return full_path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as e:
logger.warning("Konnte Artefakt nicht lesen: %s (%s)", full_path, e)
return None