feat: implement knowledge management system (spec complete, all 53 tasks done)
This commit is contained in:
@@ -784,6 +784,552 @@ class TestSharedMirrorReadOnly:
|
||||
assert len(mirror_result.mirrored_paths) >= 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 8: Full Knowledge Pipeline E2E
|
||||
# Requirements: 2.2, 2.3, 5.1, 6.1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestKnowledgePipelineE2E:
|
||||
"""E2E: Full ingestion pipeline with source configs, enrichment, and indexing."""
|
||||
|
||||
@pytest.fixture()
|
||||
def knowledge_root(self, tmp_path: Path) -> Path:
|
||||
"""Creates a knowledge-ready monorepo structure."""
|
||||
root = tmp_path / "monorepo"
|
||||
root.mkdir()
|
||||
|
||||
# Context knowledge folders
|
||||
for ctx in ("bahn", "dhive", "privat"):
|
||||
kp = root / ctx / "knowledge"
|
||||
for sub in ("inbox", "meetings", "decisions", "projects",
|
||||
"people", "references", "links"):
|
||||
(kp / sub).mkdir(parents=True)
|
||||
# Empty YAML index
|
||||
(kp / "_index.yaml").write_text(
|
||||
"version: '1.0'\nlast_updated: ''\nartifacts: []\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return root
|
||||
|
||||
def test_full_pipeline_with_markdown_source(self, knowledge_root: Path) -> None:
|
||||
"""Full pipeline run: sources.yaml → MarkdownSource → enrich → store → index."""
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
from monorepo.knowledge.ingestion.enrichment import EnrichmentAgent, EnrichmentResult
|
||||
|
||||
ctx = "bahn"
|
||||
kp = knowledge_root / ctx / "knowledge"
|
||||
|
||||
# Create a markdown file source
|
||||
source_dir = knowledge_root / ctx / "docs"
|
||||
source_dir.mkdir(parents=True)
|
||||
(source_dir / "api-design.md").write_text(
|
||||
"---\n"
|
||||
"type: decision\n"
|
||||
"title: API Design Principles\n"
|
||||
"tags: [api, design]\n"
|
||||
"---\n\n"
|
||||
"# API Design\n\nWe use RESTful conventions.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Create sources.yaml
|
||||
sources_yaml = {
|
||||
"sources": [
|
||||
{
|
||||
"type": "markdown",
|
||||
"name": "bahn-docs",
|
||||
"enabled": True,
|
||||
"params": {"directory": str(source_dir)},
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
"create_tasks": False,
|
||||
"git": {"auto_commit": False},
|
||||
},
|
||||
}
|
||||
(kp / "sources.yaml").write_text(
|
||||
yaml.dump(sources_yaml), encoding="utf-8"
|
||||
)
|
||||
|
||||
# Mock enrichment to avoid LLM calls
|
||||
mock_enrichment = MagicMock(spec=EnrichmentAgent)
|
||||
mock_enrichment.enrich.return_value = EnrichmentResult(
|
||||
title="API Design Principles",
|
||||
category="decision",
|
||||
tags=["api", "design"],
|
||||
people=[],
|
||||
projects=[],
|
||||
action_items=[],
|
||||
confidence=0.9,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=knowledge_root,
|
||||
context=ctx,
|
||||
no_commit=True,
|
||||
enrichment_agent=mock_enrichment,
|
||||
)
|
||||
|
||||
result = pipeline.run()
|
||||
|
||||
# Assertions
|
||||
assert result.processed >= 1
|
||||
assert result.context == ctx
|
||||
assert len(result.errors) == 0
|
||||
|
||||
# Index should be updated
|
||||
index_content = (kp / "_index.yaml").read_text(encoding="utf-8")
|
||||
assert "api-design" in index_content or "API" in index_content
|
||||
|
||||
def test_quick_capture_to_inbox_to_ingest_categorization(
|
||||
self, knowledge_root: Path
|
||||
) -> None:
|
||||
"""Quick Capture → Inbox → pipeline.process_inbox() → categorization."""
|
||||
from monorepo.knowledge.ingestion.capture import QuickCapture
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
from monorepo.knowledge.ingestion.enrichment import EnrichmentAgent, EnrichmentResult
|
||||
|
||||
ctx = "dhive"
|
||||
kp = knowledge_root / ctx / "knowledge"
|
||||
|
||||
# Step 1: Quick Capture a note
|
||||
capture = QuickCapture(monorepo_root=knowledge_root)
|
||||
captured_path = capture.capture(
|
||||
input_text="Meeting mit Team: Sprint Planning nächste Woche vorbereiten",
|
||||
context=ctx,
|
||||
tags=["sprint", "planning"],
|
||||
)
|
||||
|
||||
# Verify capture created file in inbox
|
||||
assert captured_path.exists()
|
||||
assert "inbox" in str(captured_path)
|
||||
content = captured_path.read_text(encoding="utf-8")
|
||||
assert "type: inbox" in content
|
||||
assert "sprint" in content
|
||||
|
||||
# Step 2: Process inbox (enrichment categorizes as meeting)
|
||||
mock_enrichment = MagicMock(spec=EnrichmentAgent)
|
||||
mock_enrichment.enrich.return_value = EnrichmentResult(
|
||||
title="Sprint Planning Vorbereitung",
|
||||
category="meeting",
|
||||
tags=["sprint", "planning", "team"],
|
||||
people=["Team"],
|
||||
projects=[],
|
||||
action_items=[],
|
||||
confidence=0.85,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=knowledge_root,
|
||||
context=ctx,
|
||||
no_commit=True,
|
||||
enrichment_agent=mock_enrichment,
|
||||
)
|
||||
|
||||
result = pipeline.process_inbox()
|
||||
|
||||
# Step 3: Verify categorization
|
||||
assert result.processed >= 1
|
||||
assert result.context == ctx
|
||||
|
||||
# Index should contain the processed artifact
|
||||
index_content = (kp / "_index.yaml").read_text(encoding="utf-8")
|
||||
assert len(index_content) > 50 # non-empty index
|
||||
|
||||
def test_confluence_source_with_mocked_api(self, knowledge_root: Path) -> None:
|
||||
"""Confluence source extraction with mocked REST API responses."""
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource, ConfluencePage
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
|
||||
source = ConfluenceSource()
|
||||
config = SourceConfig(
|
||||
type="confluence",
|
||||
name="bahn-confluence",
|
||||
params={
|
||||
"base_url": "https://confluence.bahn.de",
|
||||
"token": "fake-token",
|
||||
"pages": ["12345"],
|
||||
},
|
||||
)
|
||||
|
||||
# Mock the HTTP client interactions
|
||||
mock_page = ConfluencePage(
|
||||
page_id="12345",
|
||||
title="Architecture Decision Records",
|
||||
space_key="TEAM",
|
||||
version=3,
|
||||
content_html="<h1>ADR-001</h1><p>We use microservices architecture.</p>",
|
||||
url="https://confluence.bahn.de/pages/viewpage.action?pageId=12345",
|
||||
last_modified="2024-03-15T10:30:00Z",
|
||||
attachments=[],
|
||||
)
|
||||
|
||||
with patch.object(source, "_create_client") as mock_client_factory:
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_page.return_value = mock_page
|
||||
mock_client_factory.return_value = mock_client
|
||||
|
||||
with patch.object(source, "_collect_page_ids", return_value=["12345"]):
|
||||
extraction = source.extract(config, context="bahn")
|
||||
|
||||
# Verify extraction result
|
||||
assert len(extraction.artifacts) == 1
|
||||
assert len(extraction.errors) == 0
|
||||
|
||||
artifact = extraction.artifacts[0]
|
||||
assert "Architecture Decision Records" in artifact.metadata.title
|
||||
assert artifact.metadata.source.get("type") == "confluence"
|
||||
assert "confluence.bahn.de" in artifact.metadata.source.get("url", "")
|
||||
|
||||
def test_confluence_source_api_error_graceful(self, knowledge_root: Path) -> None:
|
||||
"""Confluence source handles API errors gracefully without crashing."""
|
||||
import urllib.error
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
|
||||
source = ConfluenceSource()
|
||||
config = SourceConfig(
|
||||
type="confluence",
|
||||
name="bahn-confluence",
|
||||
params={
|
||||
"base_url": "https://confluence.bahn.de",
|
||||
"token": "fake-token",
|
||||
"pages": ["99999"],
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(source, "_create_client") as mock_client_factory:
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_page.side_effect = urllib.error.URLError("Connection refused")
|
||||
mock_client_factory.return_value = mock_client
|
||||
|
||||
with patch.object(source, "_collect_page_ids", return_value=["99999"]):
|
||||
extraction = source.extract(config, context="bahn")
|
||||
|
||||
# Should not crash, should record error
|
||||
assert len(extraction.artifacts) == 0
|
||||
assert len(extraction.errors) >= 1
|
||||
assert extraction.errors[0].error_type == "connection"
|
||||
|
||||
def test_jira_source_with_mocked_api(self, knowledge_root: Path) -> None:
|
||||
"""Jira source extraction with mocked REST API responses."""
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
|
||||
source = JiraSource()
|
||||
config = SourceConfig(
|
||||
type="jira",
|
||||
name="bahn-jira",
|
||||
params={
|
||||
"base_url": "https://jira.bahn.de",
|
||||
"token": "fake-token",
|
||||
"username": "test-user",
|
||||
"jql": "project = ACV2 AND updatedDate > -7d",
|
||||
},
|
||||
)
|
||||
|
||||
# Mock issue data as Jira REST API v2 format
|
||||
mock_issues = [
|
||||
{
|
||||
"key": "ACV2-101",
|
||||
"fields": {
|
||||
"summary": "Implement OAuth2 login flow",
|
||||
"description": "Implement the OAuth2 flow for SSO integration.",
|
||||
"status": {"name": "In Progress"},
|
||||
"assignee": {"displayName": "Max Mustermann"},
|
||||
"reporter": {"displayName": "Anna Schmidt"},
|
||||
"project": {"key": "ACV2"},
|
||||
"comment": {"comments": [
|
||||
{"body": "Started working on this", "author": {"displayName": "Max Mustermann"}},
|
||||
]},
|
||||
"updated": "2024-03-20T14:00:00.000+0000",
|
||||
"created": "2024-03-18T09:00:00.000+0000",
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "ACV2-102",
|
||||
"fields": {
|
||||
"summary": "Fix logout redirect",
|
||||
"description": "Logout should redirect to landing page.",
|
||||
"status": {"name": "Done"},
|
||||
"assignee": {"displayName": "Anna Schmidt"},
|
||||
"reporter": {"displayName": "Max Mustermann"},
|
||||
"project": {"key": "ACV2"},
|
||||
"comment": {"comments": []},
|
||||
"updated": "2024-03-19T16:30:00.000+0000",
|
||||
"created": "2024-03-17T11:00:00.000+0000",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
with patch.object(source, "_fetch_issues", return_value=mock_issues):
|
||||
with patch.object(source, "_create_session", return_value=MagicMock()):
|
||||
extraction = source.extract(config, context="bahn")
|
||||
|
||||
# Verify extraction result
|
||||
assert len(extraction.artifacts) == 2
|
||||
assert len(extraction.errors) == 0
|
||||
|
||||
# Verify first artifact metadata
|
||||
art1 = extraction.artifacts[0]
|
||||
assert "ACV2-101" in art1.metadata.source.get("issue_key", "")
|
||||
assert art1.metadata.source.get("type") == "jira"
|
||||
assert "OAuth2" in art1.metadata.title or "OAuth2" in art1.content
|
||||
|
||||
def test_jira_source_api_error_graceful(self, knowledge_root: Path) -> None:
|
||||
"""Jira source handles API errors gracefully without crashing."""
|
||||
from monorepo.knowledge.sources.jira import JiraSource, JiraAPIError
|
||||
from monorepo.knowledge.sources.base import SourceConfig
|
||||
|
||||
source = JiraSource()
|
||||
config = SourceConfig(
|
||||
type="jira",
|
||||
name="bahn-jira",
|
||||
params={
|
||||
"base_url": "https://jira.bahn.de",
|
||||
"token": "fake-token",
|
||||
"jql": "project = ACV2",
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
source, "_fetch_issues",
|
||||
side_effect=JiraAPIError("Connection timeout", error_type="connection", retry=True),
|
||||
):
|
||||
with patch.object(source, "_create_session", return_value=MagicMock()):
|
||||
extraction = source.extract(config, context="bahn")
|
||||
|
||||
# Should not crash, should record error
|
||||
assert len(extraction.artifacts) == 0
|
||||
assert len(extraction.errors) >= 1
|
||||
assert extraction.errors[0].error_type == "connection"
|
||||
assert extraction.errors[0].retry is True
|
||||
|
||||
def test_pipeline_with_confluence_and_jira_sources(self, knowledge_root: Path) -> None:
|
||||
"""Full pipeline E2E with both Confluence and Jira sources configured."""
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
from monorepo.knowledge.ingestion.enrichment import EnrichmentAgent, EnrichmentResult
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource, ConfluencePage
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.base import SourceStrategy
|
||||
|
||||
ctx = "bahn"
|
||||
kp = knowledge_root / ctx / "knowledge"
|
||||
|
||||
# Create sources.yaml with both sources
|
||||
sources_yaml = {
|
||||
"sources": [
|
||||
{
|
||||
"type": "confluence",
|
||||
"name": "bahn-confluence",
|
||||
"enabled": True,
|
||||
"params": {
|
||||
"base_url": "https://confluence.bahn.de",
|
||||
"token": "fake-token",
|
||||
"pages": ["100"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "jira",
|
||||
"name": "bahn-jira",
|
||||
"enabled": True,
|
||||
"params": {
|
||||
"base_url": "https://jira.bahn.de",
|
||||
"token": "fake-token",
|
||||
"username": "test",
|
||||
"jql": "project = ACV2",
|
||||
},
|
||||
},
|
||||
],
|
||||
"settings": {
|
||||
"create_tasks": False,
|
||||
"git": {"auto_commit": False},
|
||||
},
|
||||
}
|
||||
(kp / "sources.yaml").write_text(
|
||||
yaml.dump(sources_yaml), encoding="utf-8"
|
||||
)
|
||||
|
||||
# Mock enrichment
|
||||
mock_enrichment = MagicMock(spec=EnrichmentAgent)
|
||||
mock_enrichment.enrich.return_value = EnrichmentResult(
|
||||
title="Test Artifact",
|
||||
category="reference",
|
||||
tags=["test"],
|
||||
people=[],
|
||||
projects=[],
|
||||
action_items=[],
|
||||
confidence=0.8,
|
||||
)
|
||||
|
||||
# Create mock strategies that return pre-built artifacts
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from monorepo.knowledge.sources.base import ExtractionResult
|
||||
from datetime import date
|
||||
|
||||
mock_confluence = MagicMock(spec=ConfluenceSource)
|
||||
mock_confluence.extract.return_value = ExtractionResult(
|
||||
artifacts=[
|
||||
Artifact(
|
||||
metadata=ArtifactMetadata(
|
||||
type="reference",
|
||||
title="Confluence Page: Architecture",
|
||||
tags=["architecture"],
|
||||
source_context=ctx,
|
||||
source={"type": "confluence", "url": "https://confluence.bahn.de/page/100"},
|
||||
created=date(2024, 3, 15),
|
||||
),
|
||||
content="# Architecture\n\nMicroservices approach.\n",
|
||||
file_path=Path("confluence-architecture.md"),
|
||||
),
|
||||
],
|
||||
errors=[],
|
||||
skipped=0,
|
||||
)
|
||||
|
||||
mock_jira = MagicMock(spec=JiraSource)
|
||||
mock_jira.extract.return_value = ExtractionResult(
|
||||
artifacts=[
|
||||
Artifact(
|
||||
metadata=ArtifactMetadata(
|
||||
type="reference",
|
||||
title="ACV2-200: Feature Request",
|
||||
tags=["jira", "feature"],
|
||||
source_context=ctx,
|
||||
source={"type": "jira", "issue_key": "ACV2-200"},
|
||||
created=date(2024, 3, 20),
|
||||
),
|
||||
content="# ACV2-200: Feature Request\n\nDetails here.\n",
|
||||
file_path=Path("acv2-200-feature-request.md"),
|
||||
),
|
||||
],
|
||||
errors=[],
|
||||
skipped=0,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=knowledge_root,
|
||||
context=ctx,
|
||||
no_commit=True,
|
||||
enrichment_agent=mock_enrichment,
|
||||
source_strategies={
|
||||
"confluence": mock_confluence,
|
||||
"jira": mock_jira,
|
||||
},
|
||||
)
|
||||
|
||||
result = pipeline.run()
|
||||
|
||||
# Both sources should contribute artifacts
|
||||
assert result.processed == 2
|
||||
assert result.context == ctx
|
||||
assert len(result.errors) == 0
|
||||
|
||||
# Index should be non-trivial
|
||||
index_content = (kp / "_index.yaml").read_text(encoding="utf-8")
|
||||
assert "artifacts" in index_content
|
||||
|
||||
def test_pipeline_partial_source_failure(self, knowledge_root: Path) -> None:
|
||||
"""Pipeline continues processing when one source fails (fail-forward)."""
|
||||
from monorepo.knowledge.ingestion.pipeline import IngestionPipeline
|
||||
from monorepo.knowledge.ingestion.enrichment import EnrichmentAgent, EnrichmentResult
|
||||
from monorepo.knowledge.sources.confluence import ConfluenceSource
|
||||
from monorepo.knowledge.sources.jira import JiraSource
|
||||
from monorepo.knowledge.sources.base import ExtractionResult, SourceError
|
||||
from monorepo.knowledge.artifact import Artifact, ArtifactMetadata
|
||||
from datetime import date
|
||||
|
||||
ctx = "bahn"
|
||||
kp = knowledge_root / ctx / "knowledge"
|
||||
|
||||
# sources.yaml with two sources – first will fail
|
||||
sources_yaml = {
|
||||
"sources": [
|
||||
{
|
||||
"type": "confluence",
|
||||
"name": "failing-source",
|
||||
"enabled": True,
|
||||
"params": {"base_url": "https://broken.example.com", "token": "x"},
|
||||
},
|
||||
{
|
||||
"type": "jira",
|
||||
"name": "working-source",
|
||||
"enabled": True,
|
||||
"params": {
|
||||
"base_url": "https://jira.bahn.de",
|
||||
"token": "x",
|
||||
"jql": "project = X",
|
||||
},
|
||||
},
|
||||
],
|
||||
"settings": {
|
||||
"create_tasks": False,
|
||||
"git": {"auto_commit": False},
|
||||
},
|
||||
}
|
||||
(kp / "sources.yaml").write_text(
|
||||
yaml.dump(sources_yaml), encoding="utf-8"
|
||||
)
|
||||
|
||||
# Mock enrichment
|
||||
mock_enrichment = MagicMock(spec=EnrichmentAgent)
|
||||
mock_enrichment.enrich.return_value = EnrichmentResult(
|
||||
title="Working Artifact",
|
||||
category="reference",
|
||||
tags=[],
|
||||
people=[],
|
||||
projects=[],
|
||||
action_items=[],
|
||||
confidence=0.8,
|
||||
)
|
||||
|
||||
# Confluence raises an exception
|
||||
mock_confluence = MagicMock(spec=ConfluenceSource)
|
||||
mock_confluence.extract.side_effect = ConnectionError("API unreachable")
|
||||
|
||||
# Jira returns a valid artifact
|
||||
mock_jira = MagicMock(spec=JiraSource)
|
||||
mock_jira.extract.return_value = ExtractionResult(
|
||||
artifacts=[
|
||||
Artifact(
|
||||
metadata=ArtifactMetadata(
|
||||
type="reference",
|
||||
title="Surviving Artifact",
|
||||
tags=["jira"],
|
||||
source_context=ctx,
|
||||
source={"type": "jira", "issue_key": "X-1"},
|
||||
created=date(2024, 3, 20),
|
||||
),
|
||||
content="# Surviving Artifact\n\nStill processed.\n",
|
||||
file_path=Path("x-1-surviving.md"),
|
||||
),
|
||||
],
|
||||
errors=[],
|
||||
skipped=0,
|
||||
)
|
||||
|
||||
pipeline = IngestionPipeline(
|
||||
monorepo_root=knowledge_root,
|
||||
context=ctx,
|
||||
no_commit=True,
|
||||
enrichment_agent=mock_enrichment,
|
||||
source_strategies={
|
||||
"confluence": mock_confluence,
|
||||
"jira": mock_jira,
|
||||
},
|
||||
)
|
||||
|
||||
result = pipeline.run()
|
||||
|
||||
# Should still process the working source
|
||||
assert result.processed >= 1
|
||||
# Should log the failure
|
||||
assert len(result.errors) >= 1
|
||||
assert any("unreachable" in e.message.lower() or "API" in e.message for e in result.errors)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test: Integration Stack Factory (create_integrated_stack)
|
||||
# Requirements: 2.2, 9.3, 9.4, 10.10
|
||||
|
||||
Reference in New Issue
Block a user