"""Unit tests for matching_method routing in both MCP matching tools. Covers task 6.6 of the ``llm-fulltext-matching`` spec: * ``matching_method=None`` → default (score) mode is used; the rendered table keeps the score columns and the META JSON tags ``"score"``. * ``matching_method="llm_fulltext"`` → the LLM matcher is invoked; the rendered table shows the ``Begründung`` column instead of score columns and META tags ``"llm_fulltext"``. * ``matching_method="bogus"`` → an error message is returned that lists both allowed values; neither the database (capacity loader / capacity by id) nor the LLM is touched. The tests cover both ``find_matching_capacities`` and ``find_matching_tasks``. _Requirements: 1.1, 1.2, 1.3, 1.4, 1.5_ """ from __future__ import annotations import json from datetime import date, datetime from typing import Any import pytest import teamlandkarte_mcp.mcp_server as mcp_mod from teamlandkarte_mcp.mcp_server import build_server from teamlandkarte_mcp.models import ( Capacity, Task, Team, TeamCompetence, TeamReference, ) _CONFIG_TEMPLATE = """ [database] host='x' port=1 username='u' password='p' backend='trino' [matching] competence_weight=0.8 role_weight=0.2 require_confirmation=false [matching.team] top_competency_weight=1.5 [cache] db_ttl_hours=1 search_ttl_minutes=60 max_size=100 [azure_openai] endpoint='https://example.openai.azure.com' api_version='2024-02-15-preview' chat_deployment='gpt-4' """.strip() # --------------------------------------------------------------------------- # Test doubles # --------------------------------------------------------------------------- class _CountingDB: """Fake DB tracking calls into the capacity/task data paths.""" def __init__(self) -> None: self.capacity_calls = 0 self.capacity_by_id_calls = 0 self.task_calls = 0 self.description_calls = 0 self.certificates_calls = 0 self.references_calls = 0 self.team_calls = 0 def test_connection(self) -> None: return def get_table_columns(self, table: str) -> list[str]: if table == "teamlandkarte_v_capacity_roles_latest": return ["name", "active", "staffing_board_relevant"] if table == "teamlandkarte_v_capacities_latest": return ["creation_date"] if table == "teamlandkarte_v_teams_latest": return [ "team_id", "ouid", "about_us", "offerings", "interests", "focus_name", ] if table == "teamlandkarte_v_teammeter_organizational_units_latest": return ["id", "name"] if table == "teamlandkarte_v_teammeter_team_competences_latest": return ["ouid", "competence_id", "top_competency"] if table == "teamlandkarte_v_team_references_latest": return ["ouid", "partner_id", "projects"] return [] def get_all_role_names(self) -> list[str]: return ["X", "Frontend Developer"] def get_all_competence_names(self) -> list[str]: return ["A", "React"] # --- capacity-search direction ------------------------------------------ def get_all_capacities_with_competences(self) -> list[Capacity]: self.capacity_calls += 1 return [ Capacity( id=1, owner_name="A", role_name="X", role_level=None, begin_date=date(2025, 1, 1), end_date=date(2025, 12, 31), competences=["A"], ) ] def get_recent_free_capacities(self, limit: int = 20) -> list[Capacity]: return [] # --- task-search direction ---------------------------------------------- def get_open_tasks(self, limit: int = 20) -> list[Task]: self.task_calls += 1 tasks = [ Task( id="T1", name="t1", title="Frontend Developer", description="React", start_date=date(2025, 5, 1), end_date=date(2025, 7, 31), created_date=datetime(2025, 1, 1), skills=["React"], ) ] return tasks if limit == 0 else tasks[:limit] def get_capacity_by_id(self, capacity_id: int | str) -> Capacity | None: self.capacity_by_id_calls += 1 return Capacity( id=int(str(capacity_id)), owner_name="A", role_name="Frontend Developer", role_level=None, begin_date=date(2025, 4, 1), end_date=date(2025, 8, 31), competences=["React"], ) # --- LLM-fulltext capacity extras --------------------------------------- def get_capacity_description(self, capacity_id: int | str) -> str | None: self.description_calls += 1 return None def get_capacity_certificates( self, capacity_id: int | str ) -> list[str]: self.certificates_calls += 1 return [] def get_capacity_references( self, capacity_id: int | str ) -> list[dict]: self.references_calls += 1 return [] # Batch variants (used by ``match_capacities``). def batch_get_capacity_descriptions( self, capacity_ids: list[Any] ) -> dict[str, str | None]: return {str(i): None for i in capacity_ids} def batch_get_capacity_certificates( self, capacity_ids: list[Any] ) -> dict[str, list[str]]: return {str(i): [] for i in capacity_ids} def batch_get_capacity_references( self, capacity_ids: list[Any] ) -> dict[str, list[dict]]: return {str(i): [] for i in capacity_ids} # --- team-search direction ---------------------------------------------- def get_all_teams(self) -> list[Team]: self.team_calls += 1 return [ Team( team_id="t1", ouid="ou-1", team_name="Team Alpha", focus_name="X", about_us="We build things.", offerings="APIs.", interests="Distributed systems.", competences=[ TeamCompetence(name="A", top_competency=True), ], references=[ TeamReference( partner_name="Partner One", projects="Project P" ), ], ) ] def get_team_by_id(self, team_id: str) -> Team | None: # pragma: no cover for t in self.get_all_teams(): if t.team_id == str(team_id): return t return None # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _result_to_text(result: Any) -> str: if isinstance(result, tuple) and len(result) == 2: content, structured = result if isinstance(structured, dict) and isinstance( structured.get("result"), str ): return structured["result"] if isinstance(content, list) and content: text = getattr(content[0], "text", None) if isinstance(text, str): return text return str(result) async def _call_tool(srv: Any, name: str, args: dict[str, Any]) -> str: return _result_to_text(await srv.call_tool(name, args)) def _extract_meta(output: str) -> dict[str, Any]: for line in output.splitlines(): if line.startswith("META="): return json.loads(line[len("META=") :]) raise AssertionError(f"no META= line in tool output:\n{output}") def _build_server_with( monkeypatch: pytest.MonkeyPatch, tmp_path, *, chat_completion: Any, ) -> tuple[Any, _CountingDB, dict[str, int]]: """Construct an MCP server backed by a counting DB and a counted LLM.""" cfg = tmp_path / "config.toml" cfg.write_text(_CONFIG_TEMPLATE, encoding="utf-8") fake_db = _CountingDB() monkeypatch.setattr( mcp_mod, "create_db_client", lambda *_a, **_k: fake_db ) llm_calls = {"count": 0} async def _wrapped(self, system, user): # noqa: ARG001 llm_calls["count"] += 1 return await chat_completion(system, user) monkeypatch.setattr( "teamlandkarte_mcp.azure.openai_client.AzureOpenAIClient.chat_completion", _wrapped, ) monkeypatch.setenv("DATA_LAKE_USERNAME", "u") monkeypatch.setenv("DATA_LAKE_PASSWORD", "p") monkeypatch.setenv("AZURE_OPENAI_LLM_API_KEY", "k") srv = build_server(str(cfg)) return srv, fake_db, llm_calls async def _score_payload(system: str, user: str) -> str: return '{"similarity": 1.0}' async def _llm_payload(system: str, user: str) -> str: return '{"category":"Top","rationale":"good fit"}' async def _fail_if_called(system: str, user: str) -> str: # pragma: no cover raise AssertionError( "chat_completion must not be called for invalid matching_method" ) # --------------------------------------------------------------------------- # Tests: find_matching_capacities # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_find_matching_capacities_default_uses_score_mode( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: """No ``matching_method`` → default ``score`` schema.""" srv, _db, _llm = _build_server_with( monkeypatch, tmp_path, chat_completion=_score_payload ) res = await _call_tool( srv, "find_matching_capacities", {"role_name": "X", "competences": ["A"]}, ) # Score-mode columns are present. assert "| Role Score |" in res assert "| Competence Score |" in res assert "| Overall Score |" in res # LLM-mode column is absent. assert "Begründung" not in res # META tags the search as score mode. meta = _extract_meta(res) assert meta.get("matching_method") == "score" @pytest.mark.asyncio async def test_find_matching_capacities_llm_mode_invokes_llm_matcher( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: """``matching_method='llm_fulltext'`` → LLM table, no score columns.""" srv, _db, llm_calls = _build_server_with( monkeypatch, tmp_path, chat_completion=_llm_payload ) res = await _call_tool( srv, "find_matching_capacities", { "role_name": "X", "competences": ["A"], "matching_method": "llm_fulltext", }, ) # The LLM was invoked at least once for the single matching capacity. assert llm_calls["count"] >= 1, "LLM matcher path was not exercised" # META reflects the LLM mode. meta = _extract_meta(res) assert meta.get("matching_method") == "llm_fulltext" # The LLM result table includes the Begründung column. assert "Begründung" in res, "LLM table missing the Begründung column" # The score columns are gone in LLM mode. assert "| Role Score |" not in res assert "| Competence Score |" not in res assert "| Overall Score |" not in res @pytest.mark.asyncio async def test_find_matching_capacities_bogus_method_returns_error( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: """Invalid ``matching_method`` → error mentioning both allowed values. Neither the capacity loader nor the LLM may be invoked. """ srv, fake_db, llm_calls = _build_server_with( monkeypatch, tmp_path, chat_completion=_fail_if_called ) res = await _call_tool( srv, "find_matching_capacities", { "role_name": "X", "competences": ["A"], "matching_method": "bogus", }, ) lowered = res.lower() assert "error" in lowered or "invalid" in lowered, res assert "score" in res assert "llm_fulltext" in res # No side effects. assert fake_db.capacity_calls == 0, ( "DB capacity loader was hit despite invalid matching_method" ) assert llm_calls["count"] == 0, ( "LLM was called despite invalid matching_method" ) # --------------------------------------------------------------------------- # Tests: find_matching_tasks # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_find_matching_tasks_llm_mode_invokes_llm_matcher( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: """``matching_method='llm_fulltext'`` for task search uses LLM matcher.""" srv, fake_db, llm_calls = _build_server_with( monkeypatch, tmp_path, chat_completion=_llm_payload ) res = await _call_tool( srv, "find_matching_tasks", {"capacity_id": 1, "matching_method": "llm_fulltext"}, ) # The LLM was invoked for the candidate task. assert llm_calls["count"] >= 1, "LLM matcher path was not exercised" # META reflects the LLM mode. meta = _extract_meta(res) assert meta.get("matching_method") == "llm_fulltext" # LLM-mode column is present, score columns absent. assert "Begründung" in res, "LLM task table missing Begründung column" assert "| Role Score |" not in res assert "| Competence Score |" not in res assert "| Overall Score |" not in res # The LLM-fulltext direction loads description/certificates/references # for the resolved capacity. assert fake_db.description_calls == 1 assert fake_db.certificates_calls == 1 assert fake_db.references_calls == 1 @pytest.mark.asyncio async def test_find_matching_tasks_bogus_method_returns_error( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: """Invalid method on task search → error, no DB/LLM access.""" srv, fake_db, llm_calls = _build_server_with( monkeypatch, tmp_path, chat_completion=_fail_if_called ) res = await _call_tool( srv, "find_matching_tasks", {"capacity_id": 1, "matching_method": "bogus"}, ) lowered = res.lower() assert "error" in lowered or "invalid" in lowered, res assert "score" in res assert "llm_fulltext" in res # No DB lookups for the rejected validation path. assert fake_db.capacity_by_id_calls == 0, ( "get_capacity_by_id was called despite invalid matching_method" ) assert fake_db.task_calls == 0, ( "get_open_tasks was called despite invalid matching_method" ) assert llm_calls["count"] == 0, ( "LLM was called despite invalid matching_method" ) # --------------------------------------------------------------------------- # Tests: find_matching_teams (Profile_Type="team") # --------------------------------------------------------------------------- # # These cases mirror the capacity-routing tests above but exercise the # team-profile path so that the routing surface is covered for both # ``Profile_Type`` values (``capacity``, ``team``) and both # ``matching_method`` values (gemocktes LLM und gemockte DB). # # _Requirements: 12.7_ @pytest.mark.asyncio async def test_find_matching_teams_score_mode_renders_team_columns( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: """``matching_method='score'`` on team search → score columns.""" srv, fake_db, llm_calls = _build_server_with( monkeypatch, tmp_path, chat_completion=_score_payload ) res = await _call_tool( srv, "find_matching_teams", { "role_name": "X", "competences": ["A"], "matching_method": "score", }, ) # Score-mode + team column layout (spec: 6.6 / 8.4). for header in ( "Team Name", "Schwerpunkt", "Top-Kompetenzen", "Role Score", "Competence Score", "Overall Score", "Category", ): assert header in res, f"missing team-search header {header!r}" assert "Begründung" not in res meta = _extract_meta(res) assert meta.get("search_type") == "team_search" assert meta.get("matching_method") == "score" # The team loader was hit; the capacity loader was not. assert fake_db.team_calls == 1 assert fake_db.capacity_calls == 0 # The mocked LLM may be invoked by the score path's role-similarity # cache, but that is not relevant here. We only assert the routing # decision via META and the column layout. _ = llm_calls @pytest.mark.asyncio async def test_find_matching_teams_llm_mode_invokes_llm_matcher( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: """``matching_method='llm_fulltext'`` on team search uses the LLM.""" srv, fake_db, llm_calls = _build_server_with( monkeypatch, tmp_path, chat_completion=_llm_payload ) res = await _call_tool( srv, "find_matching_teams", { "role_name": "X", "competences": ["A"], "matching_method": "llm_fulltext", }, ) # LLM matcher was actually invoked for the single candidate team. assert llm_calls["count"] >= 1, "LLM matcher path was not exercised" # META tags the LLM mode and the team search type. meta = _extract_meta(res) assert meta.get("search_type") == "team_search" assert meta.get("matching_method") == "llm_fulltext" # LLM-mode + team column layout (spec: 7.8 / 8.4). for header in ( "Team Name", "Schwerpunkt", "Top-Kompetenzen", "Category", "Begründung", ): assert header in res, f"missing team-search llm header {header!r}" # Score columns must be absent in LLM mode. assert "| Role Score |" not in res assert "| Competence Score |" not in res assert "| Overall Score |" not in res # Team loader was hit; capacity loader was not. assert fake_db.team_calls == 1 assert fake_db.capacity_calls == 0 @pytest.mark.asyncio async def test_find_matching_teams_bogus_method_returns_error( monkeypatch: pytest.MonkeyPatch, tmp_path, ) -> None: """Invalid ``matching_method`` on team search → error, no DB/LLM.""" srv, fake_db, llm_calls = _build_server_with( monkeypatch, tmp_path, chat_completion=_fail_if_called ) res = await _call_tool( srv, "find_matching_teams", { "role_name": "X", "competences": ["A"], "matching_method": "bogus", }, ) lowered = res.lower() assert "error" in lowered or "invalid" in lowered, res assert "score" in res assert "llm_fulltext" in res # No side effects: the team loader and the LLM are untouched. assert fake_db.team_calls == 0, ( "Team loader was hit despite invalid matching_method" ) assert llm_calls["count"] == 0, ( "LLM was called despite invalid matching_method" )