# Feature: team-profile-matching, Task 11.2 """End-to-end integration smoke test for ``find_matching_teams`` in ``llm_fulltext`` mode. Exercises the full request path through ``build_server`` with a fully-mocked ``DBClient`` and a ``MagicMock``-backed ``AzureOpenAIClient``. The mock returns a valid LLM JSON payload for two of the three teams and raises :class:`AzureAPIError` for the third so the integration verifies both happy and error paths: find_matching_teams (llm_fulltext) -> LlmFulltextMatcher.match_teams -> SearchCache.store_search (search_type="team_search", matching_method="llm_fulltext") -> get_results_by_category (renders LLM-mode columns) -> filter_search_results (role_filter) The test verifies that: * A simulated LLM error places the affected team into the response's ``## Errors`` section and the persisted ``errors`` list (Anforderung 7.6). * The persisted ``SearchCache`` payload carries ``search_type == "team_search"`` and ``matching_method == "llm_fulltext"`` (Anforderungen 8.1, 8.2, 8.3). * The Markdown output contains the LLM-mode column layout for teams (``Team Name``, ``Schwerpunkt``, ``Top-Kompetenzen``, ``Category``, ``Begründung``) (Anforderung 8.4). * Subsequent calls to ``get_results_by_category`` and ``filter_search_results`` operate on the persisted ``team_search`` entry and reflect the same markers. _Requirements: 7.1, 7.6, 8.1, 8.2, 8.4_ """ from __future__ import annotations import json from typing import Any import pytest import teamlandkarte_mcp.mcp_server as mcp_mod from teamlandkarte_mcp.azure.openai_client import AzureAPIError from teamlandkarte_mcp.cache.search_cache import SearchCache from teamlandkarte_mcp.mcp_server import build_server from teamlandkarte_mcp.models import Team, TeamCompetence, TeamReference _CONFIG_TEMPLATE = """ [database] host='x' port=1 username='u' password='p' backend='trino' [matching] competence_weight=0.8 role_weight=0.2 require_confirmation=false [matching.team] top_competency_weight=1.5 [cache] db_ttl_hours=1 search_ttl_minutes=60 max_size=100 [azure_openai] endpoint='https://example.openai.azure.com' api_version='2024-02-15-preview' chat_deployment='gpt-4' """.strip() def _result_to_text(result: Any) -> str: if isinstance(result, tuple) and len(result) == 2: content, structured = result if isinstance(structured, dict) and isinstance( structured.get("result"), str ): return structured["result"] if isinstance(content, list) and content: text = getattr(content[0], "text", None) if isinstance(text, str): return text return str(result) async def _call_tool(srv: Any, name: str, args: dict[str, Any]) -> str: return _result_to_text(await srv.call_tool(name, args)) def _extract_search_id(output: str) -> str: for line in output.splitlines(): if line.startswith("SEARCH_ID="): return line[len("SEARCH_ID="):].strip() raise AssertionError(f"no SEARCH_ID= line in tool output:\n{output}") def _extract_meta(output: str) -> dict[str, Any]: for line in output.splitlines(): if line.startswith("META="): return json.loads(line[len("META="):]) raise AssertionError(f"no META= line in tool output:\n{output}") def _make_teams() -> list[Team]: """Three deterministic teams used to exercise both LLM outcomes.""" return [ Team( team_id="t1", ouid="ou-1", team_name="Team Alpha", focus_name="Backend Developer", about_us="We build robust backends.", offerings="APIs and services.", interests="Distributed systems.", competences=[ TeamCompetence(name="python", top_competency=True), TeamCompetence(name="fastapi", top_competency=False), ], references=[ TeamReference(partner_name="DB Cargo", projects="ETL Pipeline"), ], ), Team( team_id="t2", ouid="ou-2", team_name="Team Beta", focus_name="Frontend Developer", about_us="We build slick UIs.", offerings="Web apps.", interests="Web technologies.", competences=[ TeamCompetence(name="javascript", top_competency=True), ], references=[], ), Team( team_id="t3", ouid="ou-3", team_name="Team Gamma", focus_name="Data Engineer", about_us="We build pipelines.", offerings="Data flows.", interests="Big data.", competences=[ TeamCompetence(name="spark", top_competency=False), ], references=[], ), ] class _FakeDB: """Minimal fake ``DBClient`` for the LLM team integration smoke test.""" def __init__(self, teams: list[Team]) -> None: self._teams = list(teams) def test_connection(self) -> None: return def get_table_columns(self, table: str) -> list[str]: if table == "teamlandkarte_v_capacity_roles_latest": return ["name", "active", "staffing_board_relevant"] if table == "teamlandkarte_v_capacities_latest": return ["creation_date"] if table == "teamlandkarte_v_teams_latest": return [ "team_id", "ouid", "about_us", "offerings", "interests", "focus_name", ] if table == "teamlandkarte_v_teammeter_organizational_units_latest": return ["id", "name"] if table == "teamlandkarte_v_teammeter_team_competences_latest": return ["ouid", "competence_id", "top_competency"] if table == "teamlandkarte_v_team_references_latest": return ["ouid", "partner_id", "projects"] return [] def get_all_role_names(self) -> list[str]: return ["Backend Developer", "Frontend Developer", "Data Engineer"] def get_all_competence_names(self) -> list[str]: return ["python", "fastapi", "javascript", "spark"] def get_open_tasks(self, limit: int = 20): return [] def get_all_capacities_with_competences(self): return [] def get_recent_free_capacities(self, limit: int = 20): return [] def get_all_teams(self) -> list[Team]: return list(self._teams) def get_team_by_id(self, team_id): # pragma: no cover for t in self._teams: if t.team_id == str(team_id): return t return None def _patch_capturing_store( monkeypatch: pytest.MonkeyPatch, ) -> list[dict[str, Any]]: """Capture every payload passed to ``SearchCache.store_search``.""" captured: list[dict[str, Any]] = [] original = SearchCache.store_search def _capturing(self, *, task_id, requirements, results): captured.append(results) return original( self, task_id=task_id, requirements=requirements, results=results, ) monkeypatch.setattr(SearchCache, "store_search", _capturing) return captured def _build_server_with_llm_mock( monkeypatch: pytest.MonkeyPatch, tmp_path, *, team_responses: dict[str, Any], ) -> Any: """Build a server backed by ``_FakeDB`` and a mocked LLM. ``team_responses`` maps each ``team_id`` (str) to either a JSON string (success) or an :class:`AzureAPIError` instance (simulated failure). The fake ``chat_completion`` parses the team_id out of the prompt's ``ID: `` line that :func:`_build_user_prompt_team_for_task` embeds, so the mapping is deterministic regardless of ``asyncio.gather`` scheduling. """ cfg = tmp_path / "config.toml" cfg.write_text(_CONFIG_TEMPLATE, encoding="utf-8") fake_db = _FakeDB(_make_teams()) monkeypatch.setattr( mcp_mod, "create_db_client", lambda *_a, **_k: fake_db ) async def _fake_chat(self, system: str, user: str) -> str: # noqa: ARG001 team_id = "" for line in user.splitlines(): if line.startswith("ID: "): team_id = line[len("ID: "):].strip() break outcome = team_responses.get(team_id) if isinstance(outcome, BaseException): raise outcome if outcome is None: # Defensive default: any unmapped team is reported as # ``Irrelevant`` so the test still terminates. return json.dumps( {"category": "Irrelevant", "rationale": "default"}, ensure_ascii=False, ) return outcome monkeypatch.setattr( "teamlandkarte_mcp.azure.openai_client." "AzureOpenAIClient.chat_completion", _fake_chat, ) monkeypatch.setenv("DATA_LAKE_USERNAME", "u") monkeypatch.setenv("DATA_LAKE_PASSWORD", "p") monkeypatch.setenv("AZURE_OPENAI_LLM_API_KEY", "k") return build_server(str(cfg)) @pytest.mark.asyncio async def test_e2e_find_matching_teams_llm_fulltext_full_path( monkeypatch: pytest.MonkeyPatch, tmp_path ) -> None: """End-to-end smoke test for ``find_matching_teams`` (llm_fulltext). Validates: requirements 7.1 (LLM-categorisation per team), 7.6 (LLM error surfaces in ``errors`` list), 8.1, 8.2 (cache markers) and 8.4 (LLM-mode column layout). """ # Two valid LLM payloads and one simulated failure. team_responses: dict[str, Any] = { "t1": json.dumps( {"category": "Top", "rationale": "Backend match."}, ensure_ascii=False, ), "t2": json.dumps( {"category": "Partial", "rationale": "Frontend overlap."}, ensure_ascii=False, ), "t3": AzureAPIError("LLM call failed"), } captured = _patch_capturing_store(monkeypatch) srv = _build_server_with_llm_mock( monkeypatch, tmp_path, team_responses=team_responses ) # ------------------------------------------------------------------ # 1) find_matching_teams (llm_fulltext mode) # ------------------------------------------------------------------ out = await _call_tool( srv, "find_matching_teams", { "role_name": "Backend Developer", "competences": ["python"], "matching_method": "llm_fulltext", }, ) # SEARCH_ID + META markers (Anforderungen 8.1, 8.3). search_id = _extract_search_id(out) meta = _extract_meta(out) assert meta.get("search_id") == search_id assert meta.get("search_type") == "team_search" assert meta.get("matching_method") == "llm_fulltext" # The errored team (t3) must be visible in the rendered output via # the ``## Errors`` section (Anforderung 7.6). assert "## Errors" in out, out assert "t3" in out, out # Team t3 must NOT appear in the rendered category table because # errored teams never end up in by_category. # (We check the persisted payload below for the strict invariant.) # The team-search LLM column layout must be present # (Anforderung 8.4). for header in ( "Team Name", "Schwerpunkt", "Top-Kompetenzen", "Category", "Begründung", ): assert header in out, ( f"missing team-search llm-mode header {header!r} in:\n{out}" ) # The persisted SearchCache payload carries the team_search markers # and the errors list (Anforderungen 7.6, 8.2). assert captured, "SearchCache.store_search was never called" payload = captured[-1] assert payload["search_type"] == "team_search" assert payload["matching_method"] == "llm_fulltext" errors = payload.get("errors") or [] error_ids = {e["item_id"] for e in errors} assert error_ids == {"t3"}, errors by_category = payload.get("by_category") or {} flat_items = [item for items in by_category.values() for item in items] flat_ids = {str(it.get("team_id")) for it in flat_items} # The two successful teams are present, the errored team is not. assert flat_ids == {"t1", "t2"}, flat_ids # by_category must NOT contain the errored team. assert "t3" not in flat_ids # Each persisted item carries category + rationale fields. for item in flat_items: assert "team_id" in item assert "category" in item assert "rationale" in item assert isinstance(item["rationale"], str) assert item["rationale"].strip() # ------------------------------------------------------------------ # 2) get_results_by_category — renders the LLM-mode team table. # ------------------------------------------------------------------ default_category = str(meta.get("default_category") or "Top") cat_out = await _call_tool( srv, "get_results_by_category", { "search_id": search_id, "category": default_category, "page": 1, "page_size": 20, }, ) cat_meta = _extract_meta(cat_out) assert cat_meta.get("search_type") == "team_search" assert cat_meta.get("matching_method") == "llm_fulltext" assert cat_meta.get("search_id") == search_id for header in ( "Team Name", "Schwerpunkt", "Top-Kompetenzen", "Category", "Begründung", ): assert header in cat_out, ( f"missing team-search llm-mode header {header!r} in " f"get_results_by_category output:\n{cat_out}" ) # Score-only columns must NOT appear in LLM-mode tables. assert "Role Score" not in cat_out assert "Overall Score" not in cat_out # ------------------------------------------------------------------ # 3) filter_search_results — round-trip on the team_search entry. # ------------------------------------------------------------------ filtered_out = await _call_tool( srv, "filter_search_results", { "search_id": search_id, "role_filter": "Backend", }, ) assert "FILTER_ID=" in filtered_out, filtered_out filter_meta = _extract_meta(filtered_out) assert filter_meta.get("search_type") == "team_search" assert filter_meta.get("matching_method") == "llm_fulltext" assert filter_meta.get("status") == "ok" # Only Team Alpha's focus_name matches "Backend"; t3 is in errors # (not in by_category), so the filtered total must be exactly 1. assert filter_meta.get("total") == 1, filter_meta assert "## Applied Filters" in filtered_out assert "role_filter" in filtered_out assert "Backend" in filtered_out # LLM-mode preview must include the Begründung column. assert "Begründung" in filtered_out assert "Filtered total results: 1" in filtered_out