"""Unit tests for ``LlmFulltextMatcher`` with a mocked ``AzureOpenAIClient``. These tests cover task 3.10 of the ``llm-fulltext-matching`` spec: * success case (valid category + rationale), * invalid JSON, * valid JSON but not an object, * valid JSON with an unknown category, * ``AzureAPIError`` raised by ``chat_completion``, * generic exception raised by ``chat_completion``, * mixed batch (deterministic ordering, sorting, error separation), * task-direction success (``match_tasks``). Validates: Requirements 5.4, 5.5, 5.6, 5.7, 5.8, 6.4, 6.5, 6.6, 6.7, 6.8. """ from __future__ import annotations import asyncio import json import re from datetime import datetime from typing import Any import pytest from teamlandkarte_mcp.azure.openai_client import AzureAPIError from teamlandkarte_mcp.matching.llm_fulltext_matcher import LlmFulltextMatcher from teamlandkarte_mcp.matching.profiles import ( CapacityProfile, CapacityReferenceEntry, TaskProfile, ) from teamlandkarte_mcp.models import Capacity, Task # --------------------------------------------------------------------------- # Test doubles # --------------------------------------------------------------------------- class _StubDb: """Minimal ``DBClient`` double returning empty extras for every id.""" def batch_get_capacity_descriptions( self, capacity_ids: list[Any] ) -> dict[str, str | None]: return {str(i): None for i in capacity_ids} def batch_get_capacity_certificates( self, capacity_ids: list[Any] ) -> dict[str, list[str]]: return {str(i): [] for i in capacity_ids} def batch_get_capacity_references( self, capacity_ids: list[Any] ) -> dict[str, list[dict]]: return {str(i): [] for i in capacity_ids} class _StaticLlm: """Returns the same payload (or raises) for every ``chat_completion`` call. Use for single-item tests where the prompt content is irrelevant. """ def __init__( self, payload: str | None = None, exc: BaseException | None = None, ) -> None: self._payload = payload self._exc = exc async def chat_completion(self, system: str, user: str) -> str: if self._exc is not None: raise self._exc assert self._payload is not None return self._payload class _MapLlm: """Returns a payload (or raises) keyed by the ``ID:`` line of the prompt. The matcher emits a line ``ID: `` in the user prompt for both directions. The mock parses that line to look up the response, which keeps the test deterministic regardless of iteration order. """ _ID_RE = re.compile(r"ID:\s*(\S+)") def __init__( self, id_to_payload: dict[str, str], id_to_exception: dict[str, BaseException] | None = None, ) -> None: self._payloads = id_to_payload self._exceptions = id_to_exception or {} async def chat_completion(self, system: str, user: str) -> str: match = self._ID_RE.search(user) cid = match.group(1) if match else "unknown" if cid in self._exceptions: raise self._exceptions[cid] return self._payloads.get( cid, '{"category":"Irrelevant","rationale":"none"}' ) # --------------------------------------------------------------------------- # Factories # --------------------------------------------------------------------------- def _cap(cid: int) -> Capacity: return Capacity( id=cid, owner_name=f"o{cid}", role_name="R", role_level=None, begin_date=None, end_date=None, competences=[], ) def _task_profile() -> TaskProfile: return TaskProfile(id="t1", title="T", description="D", skills=[]) def _capacity_profile() -> CapacityProfile: return CapacityProfile( id="42", owner_name="alice", role_name="Engineer", competences=["python", "sql"], description="senior backend engineer", references=[ CapacityReferenceEntry(partner_name="ACME", projects="proj-x"), CapacityReferenceEntry(partner_name="", projects="internal"), ], certificates=["cert-A"], ) def _make_task(tid: str) -> Task: return Task( id=tid, name=f"name-{tid}", title=f"Task {tid}", description=f"desc {tid}", start_date=None, end_date=None, created_date=datetime(2025, 1, 1), skills=["python"], ) _CATEGORIES = ("Top", "Good", "Partial", "Low", "Irrelevant") # --------------------------------------------------------------------------- # Tests: match_capacities # --------------------------------------------------------------------------- def test_success_case_valid_category_and_rationale() -> None: """Valid JSON with a canonical category lands in the right bucket.""" payload = json.dumps({"category": "Top", "rationale": "perfect fit"}) matcher = LlmFulltextMatcher(db=_StubDb(), client=_StaticLlm(payload)) result = asyncio.run( matcher.match_capacities( task_profile=_task_profile(), capacities=[_cap(7)], ) ) assert result.errors == [] assert result.by_category["Top"] and len(result.by_category["Top"]) == 1 item = result.by_category["Top"][0] assert item.item_id == "7" assert item.category == "Top" assert item.rationale == "perfect fit" # Other buckets are empty but present. for cat in ("Good", "Partial", "Low", "Irrelevant"): assert result.by_category[cat] == [] def test_invalid_json_routes_to_errors() -> None: """Non-JSON LLM output is recorded as an error, not a result.""" matcher = LlmFulltextMatcher( db=_StubDb(), client=_StaticLlm("not json at all") ) result = asyncio.run( matcher.match_capacities( task_profile=_task_profile(), capacities=[_cap(7)], ) ) assert len(result.errors) == 1 err = result.errors[0] assert err.item_id == "7" assert err.error.startswith("invalid JSON: ") # No item is categorized. for cat in _CATEGORIES: assert result.by_category[cat] == [] def test_valid_json_not_an_object_routes_to_errors() -> None: """JSON that is not a dict is treated as an invalid payload.""" payload = json.dumps("just a string") matcher = LlmFulltextMatcher(db=_StubDb(), client=_StaticLlm(payload)) result = asyncio.run( matcher.match_capacities( task_profile=_task_profile(), capacities=[_cap(9)], ) ) assert len(result.errors) == 1 err = result.errors[0] assert err.item_id == "9" assert err.error.startswith("invalid JSON: not an object") for cat in _CATEGORIES: assert result.by_category[cat] == [] def test_unknown_category_is_routed_to_irrelevant_with_hint() -> None: """Unknown categories are remapped to ``Irrelevant`` with a hint.""" payload = json.dumps({"category": "Bogus", "rationale": "rea"}) matcher = LlmFulltextMatcher(db=_StubDb(), client=_StaticLlm(payload)) result = asyncio.run( matcher.match_capacities( task_profile=_task_profile(), capacities=[_cap(3)], ) ) assert result.errors == [] assert len(result.by_category["Irrelevant"]) == 1 item = result.by_category["Irrelevant"][0] assert item.item_id == "3" assert item.category == "Irrelevant" assert "rea" in item.rationale assert "[Hinweis: ungültige LLM-Kategorie:" in item.rationale for cat in ("Top", "Good", "Partial", "Low"): assert result.by_category[cat] == [] def test_azure_api_error_is_recorded_in_errors() -> None: """``AzureAPIError`` from ``chat_completion`` lands in the error list.""" matcher = LlmFulltextMatcher( db=_StubDb(), client=_StaticLlm(exc=AzureAPIError("boom")) ) result = asyncio.run( matcher.match_capacities( task_profile=_task_profile(), capacities=[_cap(11)], ) ) assert len(result.errors) == 1 err = result.errors[0] assert err.item_id == "11" assert err.error == "AzureAPIError: boom" for cat in _CATEGORIES: assert result.by_category[cat] == [] def test_generic_exception_is_recorded_in_errors() -> None: """A non-Azure exception is also routed to the error list.""" matcher = LlmFulltextMatcher( db=_StubDb(), client=_StaticLlm(exc=RuntimeError("nope")) ) result = asyncio.run( matcher.match_capacities( task_profile=_task_profile(), capacities=[_cap(13)], ) ) assert len(result.errors) == 1 err = result.errors[0] assert err.item_id == "13" assert err.error == "RuntimeError: nope" for cat in _CATEGORIES: assert result.by_category[cat] == [] def test_mixed_batch_orders_buckets_lexicographically() -> None: """Mixed batch: success, success, JSON error, success → sorted output.""" id_to_payload = { "30": json.dumps({"category": "Top", "rationale": "t30"}), "10": json.dumps({"category": "Top", "rationale": "t10"}), "20": "not json", "40": json.dumps({"category": "Good", "rationale": "g40"}), } matcher = LlmFulltextMatcher( db=_StubDb(), client=_MapLlm(id_to_payload) ) capacities = [_cap(30), _cap(10), _cap(20), _cap(40)] result = asyncio.run( matcher.match_capacities( task_profile=_task_profile(), capacities=capacities, ) ) # Lexicographically ascending within each category. top_ids = [it.item_id for it in result.by_category["Top"]] assert top_ids == ["10", "30"] good_ids = [it.item_id for it in result.by_category["Good"]] assert good_ids == ["40"] for cat in ("Partial", "Low", "Irrelevant"): assert result.by_category[cat] == [] # All five canonical buckets are present in the dict. assert set(result.by_category.keys()) == set(_CATEGORIES) # The invalid-JSON item is only in the error list. assert len(result.errors) == 1 assert result.errors[0].item_id == "20" assert result.errors[0].error.startswith("invalid JSON: ") # --------------------------------------------------------------------------- # Tests: match_tasks # --------------------------------------------------------------------------- def test_match_tasks_success_case() -> None: """``match_tasks`` mirrors the capacity direction for the success path.""" payload = json.dumps({"category": "Good", "rationale": "task fit"}) matcher = LlmFulltextMatcher(db=_StubDb(), client=_StaticLlm(payload)) task = _make_task("00T123") result = asyncio.run( matcher.match_tasks( capacity_profile=_capacity_profile(), tasks=[task], ) ) assert result.errors == [] assert len(result.by_category["Good"]) == 1 item = result.by_category["Good"][0] assert item.item_id == "00T123" assert item.category == "Good" assert item.rationale == "task fit" for cat in ("Top", "Partial", "Low", "Irrelevant"): assert result.by_category[cat] == [] if __name__ == "__main__": # pragma: no cover raise SystemExit(pytest.main([__file__, "-x", "--tb=short", "-q"]))