Migrate all repos into monorepo context folders
Bahn: aisupport, Analyse-O2C-C2S, awesome-bahn-mcp-servers, beam-mcp,
Confluence_Bot, db-planet-mcp-server, O2C-Harness, project-audit,
Projekt-KIQ-HP, teamlandkarte-mcp
Dhive: Jury-Voting
Privat: CV, NoteGraph (NOTE: NoteGraph needs complete redo after consolidation)
Shared: AI-Orchestrator, OrgMyLife, power_skills_and_more
Shared/references: symphony (read-only)
Bahn repos remain available as independent remotes - this monorepo
pulls them in via subtree, the originals are untouched.
This commit is contained in:
@@ -0,0 +1,363 @@
|
||||
"""Unit tests for ``LlmFulltextMatcher`` with a mocked ``AzureOpenAIClient``.
|
||||
|
||||
These tests cover task 3.10 of the ``llm-fulltext-matching`` spec:
|
||||
|
||||
* success case (valid category + rationale),
|
||||
* invalid JSON,
|
||||
* valid JSON but not an object,
|
||||
* valid JSON with an unknown category,
|
||||
* ``AzureAPIError`` raised by ``chat_completion``,
|
||||
* generic exception raised by ``chat_completion``,
|
||||
* mixed batch (deterministic ordering, sorting, error separation),
|
||||
* task-direction success (``match_tasks``).
|
||||
|
||||
Validates: Requirements 5.4, 5.5, 5.6, 5.7, 5.8, 6.4, 6.5, 6.6, 6.7, 6.8.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from teamlandkarte_mcp.azure.openai_client import AzureAPIError
|
||||
from teamlandkarte_mcp.matching.llm_fulltext_matcher import LlmFulltextMatcher
|
||||
from teamlandkarte_mcp.matching.profiles import (
|
||||
CapacityProfile,
|
||||
CapacityReferenceEntry,
|
||||
TaskProfile,
|
||||
)
|
||||
from teamlandkarte_mcp.models import Capacity, Task
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test doubles
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _StubDb:
|
||||
"""Minimal ``DBClient`` double returning empty extras for every id."""
|
||||
|
||||
def batch_get_capacity_descriptions(
|
||||
self, capacity_ids: list[Any]
|
||||
) -> dict[str, str | None]:
|
||||
return {str(i): None for i in capacity_ids}
|
||||
|
||||
def batch_get_capacity_certificates(
|
||||
self, capacity_ids: list[Any]
|
||||
) -> dict[str, list[str]]:
|
||||
return {str(i): [] for i in capacity_ids}
|
||||
|
||||
def batch_get_capacity_references(
|
||||
self, capacity_ids: list[Any]
|
||||
) -> dict[str, list[dict]]:
|
||||
return {str(i): [] for i in capacity_ids}
|
||||
|
||||
|
||||
class _StaticLlm:
|
||||
"""Returns the same payload (or raises) for every ``chat_completion`` call.
|
||||
|
||||
Use for single-item tests where the prompt content is irrelevant.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
payload: str | None = None,
|
||||
exc: BaseException | None = None,
|
||||
) -> None:
|
||||
self._payload = payload
|
||||
self._exc = exc
|
||||
|
||||
async def chat_completion(self, system: str, user: str) -> str:
|
||||
if self._exc is not None:
|
||||
raise self._exc
|
||||
assert self._payload is not None
|
||||
return self._payload
|
||||
|
||||
|
||||
class _MapLlm:
|
||||
"""Returns a payload (or raises) keyed by the ``ID:`` line of the prompt.
|
||||
|
||||
The matcher emits a line ``ID: <id>`` in the user prompt for both
|
||||
directions. The mock parses that line to look up the response, which
|
||||
keeps the test deterministic regardless of iteration order.
|
||||
"""
|
||||
|
||||
_ID_RE = re.compile(r"ID:\s*(\S+)")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
id_to_payload: dict[str, str],
|
||||
id_to_exception: dict[str, BaseException] | None = None,
|
||||
) -> None:
|
||||
self._payloads = id_to_payload
|
||||
self._exceptions = id_to_exception or {}
|
||||
|
||||
async def chat_completion(self, system: str, user: str) -> str:
|
||||
match = self._ID_RE.search(user)
|
||||
cid = match.group(1) if match else "unknown"
|
||||
if cid in self._exceptions:
|
||||
raise self._exceptions[cid]
|
||||
return self._payloads.get(
|
||||
cid, '{"category":"Irrelevant","rationale":"none"}'
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Factories
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cap(cid: int) -> Capacity:
|
||||
return Capacity(
|
||||
id=cid,
|
||||
owner_name=f"o{cid}",
|
||||
role_name="R",
|
||||
role_level=None,
|
||||
begin_date=None,
|
||||
end_date=None,
|
||||
competences=[],
|
||||
)
|
||||
|
||||
|
||||
def _task_profile() -> TaskProfile:
|
||||
return TaskProfile(id="t1", title="T", description="D", skills=[])
|
||||
|
||||
|
||||
def _capacity_profile() -> CapacityProfile:
|
||||
return CapacityProfile(
|
||||
id="42",
|
||||
owner_name="alice",
|
||||
role_name="Engineer",
|
||||
competences=["python", "sql"],
|
||||
description="senior backend engineer",
|
||||
references=[
|
||||
CapacityReferenceEntry(partner_name="ACME", projects="proj-x"),
|
||||
CapacityReferenceEntry(partner_name="", projects="internal"),
|
||||
],
|
||||
certificates=["cert-A"],
|
||||
)
|
||||
|
||||
|
||||
def _make_task(tid: str) -> Task:
|
||||
return Task(
|
||||
id=tid,
|
||||
name=f"name-{tid}",
|
||||
title=f"Task {tid}",
|
||||
description=f"desc {tid}",
|
||||
start_date=None,
|
||||
end_date=None,
|
||||
created_date=datetime(2025, 1, 1),
|
||||
skills=["python"],
|
||||
)
|
||||
|
||||
|
||||
_CATEGORIES = ("Top", "Good", "Partial", "Low", "Irrelevant")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: match_capacities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_success_case_valid_category_and_rationale() -> None:
|
||||
"""Valid JSON with a canonical category lands in the right bucket."""
|
||||
payload = json.dumps({"category": "Top", "rationale": "perfect fit"})
|
||||
matcher = LlmFulltextMatcher(db=_StubDb(), client=_StaticLlm(payload))
|
||||
|
||||
result = asyncio.run(
|
||||
matcher.match_capacities(
|
||||
task_profile=_task_profile(),
|
||||
capacities=[_cap(7)],
|
||||
)
|
||||
)
|
||||
|
||||
assert result.errors == []
|
||||
assert result.by_category["Top"] and len(result.by_category["Top"]) == 1
|
||||
item = result.by_category["Top"][0]
|
||||
assert item.item_id == "7"
|
||||
assert item.category == "Top"
|
||||
assert item.rationale == "perfect fit"
|
||||
# Other buckets are empty but present.
|
||||
for cat in ("Good", "Partial", "Low", "Irrelevant"):
|
||||
assert result.by_category[cat] == []
|
||||
|
||||
|
||||
def test_invalid_json_routes_to_errors() -> None:
|
||||
"""Non-JSON LLM output is recorded as an error, not a result."""
|
||||
matcher = LlmFulltextMatcher(
|
||||
db=_StubDb(), client=_StaticLlm("not json at all")
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
matcher.match_capacities(
|
||||
task_profile=_task_profile(),
|
||||
capacities=[_cap(7)],
|
||||
)
|
||||
)
|
||||
|
||||
assert len(result.errors) == 1
|
||||
err = result.errors[0]
|
||||
assert err.item_id == "7"
|
||||
assert err.error.startswith("invalid JSON: ")
|
||||
# No item is categorized.
|
||||
for cat in _CATEGORIES:
|
||||
assert result.by_category[cat] == []
|
||||
|
||||
|
||||
def test_valid_json_not_an_object_routes_to_errors() -> None:
|
||||
"""JSON that is not a dict is treated as an invalid payload."""
|
||||
payload = json.dumps("just a string")
|
||||
matcher = LlmFulltextMatcher(db=_StubDb(), client=_StaticLlm(payload))
|
||||
|
||||
result = asyncio.run(
|
||||
matcher.match_capacities(
|
||||
task_profile=_task_profile(),
|
||||
capacities=[_cap(9)],
|
||||
)
|
||||
)
|
||||
|
||||
assert len(result.errors) == 1
|
||||
err = result.errors[0]
|
||||
assert err.item_id == "9"
|
||||
assert err.error.startswith("invalid JSON: not an object")
|
||||
for cat in _CATEGORIES:
|
||||
assert result.by_category[cat] == []
|
||||
|
||||
|
||||
def test_unknown_category_is_routed_to_irrelevant_with_hint() -> None:
|
||||
"""Unknown categories are remapped to ``Irrelevant`` with a hint."""
|
||||
payload = json.dumps({"category": "Bogus", "rationale": "rea"})
|
||||
matcher = LlmFulltextMatcher(db=_StubDb(), client=_StaticLlm(payload))
|
||||
|
||||
result = asyncio.run(
|
||||
matcher.match_capacities(
|
||||
task_profile=_task_profile(),
|
||||
capacities=[_cap(3)],
|
||||
)
|
||||
)
|
||||
|
||||
assert result.errors == []
|
||||
assert len(result.by_category["Irrelevant"]) == 1
|
||||
item = result.by_category["Irrelevant"][0]
|
||||
assert item.item_id == "3"
|
||||
assert item.category == "Irrelevant"
|
||||
assert "rea" in item.rationale
|
||||
assert "[Hinweis: ungültige LLM-Kategorie:" in item.rationale
|
||||
for cat in ("Top", "Good", "Partial", "Low"):
|
||||
assert result.by_category[cat] == []
|
||||
|
||||
|
||||
def test_azure_api_error_is_recorded_in_errors() -> None:
|
||||
"""``AzureAPIError`` from ``chat_completion`` lands in the error list."""
|
||||
matcher = LlmFulltextMatcher(
|
||||
db=_StubDb(), client=_StaticLlm(exc=AzureAPIError("boom"))
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
matcher.match_capacities(
|
||||
task_profile=_task_profile(),
|
||||
capacities=[_cap(11)],
|
||||
)
|
||||
)
|
||||
|
||||
assert len(result.errors) == 1
|
||||
err = result.errors[0]
|
||||
assert err.item_id == "11"
|
||||
assert err.error == "AzureAPIError: boom"
|
||||
for cat in _CATEGORIES:
|
||||
assert result.by_category[cat] == []
|
||||
|
||||
|
||||
def test_generic_exception_is_recorded_in_errors() -> None:
|
||||
"""A non-Azure exception is also routed to the error list."""
|
||||
matcher = LlmFulltextMatcher(
|
||||
db=_StubDb(), client=_StaticLlm(exc=RuntimeError("nope"))
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
matcher.match_capacities(
|
||||
task_profile=_task_profile(),
|
||||
capacities=[_cap(13)],
|
||||
)
|
||||
)
|
||||
|
||||
assert len(result.errors) == 1
|
||||
err = result.errors[0]
|
||||
assert err.item_id == "13"
|
||||
assert err.error == "RuntimeError: nope"
|
||||
for cat in _CATEGORIES:
|
||||
assert result.by_category[cat] == []
|
||||
|
||||
|
||||
def test_mixed_batch_orders_buckets_lexicographically() -> None:
|
||||
"""Mixed batch: success, success, JSON error, success → sorted output."""
|
||||
id_to_payload = {
|
||||
"30": json.dumps({"category": "Top", "rationale": "t30"}),
|
||||
"10": json.dumps({"category": "Top", "rationale": "t10"}),
|
||||
"20": "not json",
|
||||
"40": json.dumps({"category": "Good", "rationale": "g40"}),
|
||||
}
|
||||
matcher = LlmFulltextMatcher(
|
||||
db=_StubDb(), client=_MapLlm(id_to_payload)
|
||||
)
|
||||
capacities = [_cap(30), _cap(10), _cap(20), _cap(40)]
|
||||
|
||||
result = asyncio.run(
|
||||
matcher.match_capacities(
|
||||
task_profile=_task_profile(),
|
||||
capacities=capacities,
|
||||
)
|
||||
)
|
||||
|
||||
# Lexicographically ascending within each category.
|
||||
top_ids = [it.item_id for it in result.by_category["Top"]]
|
||||
assert top_ids == ["10", "30"]
|
||||
good_ids = [it.item_id for it in result.by_category["Good"]]
|
||||
assert good_ids == ["40"]
|
||||
for cat in ("Partial", "Low", "Irrelevant"):
|
||||
assert result.by_category[cat] == []
|
||||
|
||||
# All five canonical buckets are present in the dict.
|
||||
assert set(result.by_category.keys()) == set(_CATEGORIES)
|
||||
|
||||
# The invalid-JSON item is only in the error list.
|
||||
assert len(result.errors) == 1
|
||||
assert result.errors[0].item_id == "20"
|
||||
assert result.errors[0].error.startswith("invalid JSON: ")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: match_tasks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_match_tasks_success_case() -> None:
|
||||
"""``match_tasks`` mirrors the capacity direction for the success path."""
|
||||
payload = json.dumps({"category": "Good", "rationale": "task fit"})
|
||||
matcher = LlmFulltextMatcher(db=_StubDb(), client=_StaticLlm(payload))
|
||||
task = _make_task("00T123")
|
||||
|
||||
result = asyncio.run(
|
||||
matcher.match_tasks(
|
||||
capacity_profile=_capacity_profile(),
|
||||
tasks=[task],
|
||||
)
|
||||
)
|
||||
|
||||
assert result.errors == []
|
||||
assert len(result.by_category["Good"]) == 1
|
||||
item = result.by_category["Good"][0]
|
||||
assert item.item_id == "00T123"
|
||||
assert item.category == "Good"
|
||||
assert item.rationale == "task fit"
|
||||
for cat in ("Top", "Partial", "Low", "Irrelevant"):
|
||||
assert result.by_category[cat] == []
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(pytest.main([__file__, "-x", "--tb=short", "-q"]))
|
||||
Reference in New Issue
Block a user