Orchestrator/bahn/teamlandkarte-mcp/tests/test_team_integration_llm_fulltext.py

# Feature: team-profile-matching, Task 11.2
"""End-to-end integration smoke test for ``find_matching_teams`` in
``llm_fulltext`` mode.

Exercises the full request path through ``build_server`` with a
fully-mocked ``DBClient`` and a ``MagicMock``-backed
``AzureOpenAIClient``. The mock returns a valid LLM JSON payload for
two of the three teams and raises :class:`AzureAPIError` for the third
so the integration verifies both happy and error paths:

    find_matching_teams (llm_fulltext)
      -> LlmFulltextMatcher.match_teams
      -> SearchCache.store_search (search_type="team_search",
                                   matching_method="llm_fulltext")
      -> get_results_by_category (renders LLM-mode columns)
      -> filter_search_results (role_filter)

The test verifies that:

* A simulated LLM error places the affected team into the response's
  ``## Errors`` section and the persisted ``errors`` list (Anforderung
  7.6).
* The persisted ``SearchCache`` payload carries
  ``search_type == "team_search"`` and
  ``matching_method == "llm_fulltext"`` (Anforderungen 8.1, 8.2, 8.3).
* The Markdown output contains the LLM-mode column layout for teams
  (``Team Name``, ``Schwerpunkt``, ``Top-Kompetenzen``, ``Category``,
  ``Begründung``) (Anforderung 8.4).
* Subsequent calls to ``get_results_by_category`` and
  ``filter_search_results`` operate on the persisted ``team_search``
  entry and reflect the same markers.

_Requirements: 7.1, 7.6, 8.1, 8.2, 8.4_
"""

from __future__ import annotations

import json
from typing import Any

import pytest

import teamlandkarte_mcp.mcp_server as mcp_mod
from teamlandkarte_mcp.azure.openai_client import AzureAPIError
from teamlandkarte_mcp.cache.search_cache import SearchCache
from teamlandkarte_mcp.mcp_server import build_server
from teamlandkarte_mcp.models import Team, TeamCompetence, TeamReference


_CONFIG_TEMPLATE = """
[database]
host='x'
port=1
username='u'
password='p'
backend='trino'

[matching]
competence_weight=0.8
role_weight=0.2
require_confirmation=false

[matching.team]
top_competency_weight=1.5

[cache]
db_ttl_hours=1
search_ttl_minutes=60
max_size=100

[azure_openai]
endpoint='https://example.openai.azure.com'
api_version='2024-02-15-preview'
chat_deployment='gpt-4'
""".strip()


def _result_to_text(result: Any) -> str:
    if isinstance(result, tuple) and len(result) == 2:
        content, structured = result
        if isinstance(structured, dict) and isinstance(
            structured.get("result"), str
        ):
            return structured["result"]
        if isinstance(content, list) and content:
            text = getattr(content[0], "text", None)
            if isinstance(text, str):
                return text
    return str(result)


async def _call_tool(srv: Any, name: str, args: dict[str, Any]) -> str:
    return _result_to_text(await srv.call_tool(name, args))


def _extract_search_id(output: str) -> str:
    for line in output.splitlines():
        if line.startswith("SEARCH_ID="):
            return line[len("SEARCH_ID="):].strip()
    raise AssertionError(f"no SEARCH_ID= line in tool output:\n{output}")


def _extract_meta(output: str) -> dict[str, Any]:
    for line in output.splitlines():
        if line.startswith("META="):
            return json.loads(line[len("META="):])
    raise AssertionError(f"no META= line in tool output:\n{output}")


def _make_teams() -> list[Team]:
    """Three deterministic teams used to exercise both LLM outcomes."""
    return [
        Team(
            team_id="t1",
            ouid="ou-1",
            team_name="Team Alpha",
            focus_name="Backend Developer",
            about_us="We build robust backends.",
            offerings="APIs and services.",
            interests="Distributed systems.",
            competences=[
                TeamCompetence(name="python", top_competency=True),
                TeamCompetence(name="fastapi", top_competency=False),
            ],
            references=[
                TeamReference(partner_name="DB Cargo", projects="ETL Pipeline"),
            ],
        ),
        Team(
            team_id="t2",
            ouid="ou-2",
            team_name="Team Beta",
            focus_name="Frontend Developer",
            about_us="We build slick UIs.",
            offerings="Web apps.",
            interests="Web technologies.",
            competences=[
                TeamCompetence(name="javascript", top_competency=True),
            ],
            references=[],
        ),
        Team(
            team_id="t3",
            ouid="ou-3",
            team_name="Team Gamma",
            focus_name="Data Engineer",
            about_us="We build pipelines.",
            offerings="Data flows.",
            interests="Big data.",
            competences=[
                TeamCompetence(name="spark", top_competency=False),
            ],
            references=[],
        ),
    ]


class _FakeDB:
    """Minimal fake ``DBClient`` for the LLM team integration smoke test."""

    def __init__(self, teams: list[Team]) -> None:
        self._teams = list(teams)

    def test_connection(self) -> None:
        return

    def get_table_columns(self, table: str) -> list[str]:
        if table == "teamlandkarte_v_capacity_roles_latest":
            return ["name", "active", "staffing_board_relevant"]
        if table == "teamlandkarte_v_capacities_latest":
            return ["creation_date"]
        if table == "teamlandkarte_v_teams_latest":
            return [
                "team_id",
                "ouid",
                "about_us",
                "offerings",
                "interests",
                "focus_name",
            ]
        if table == "teamlandkarte_v_teammeter_organizational_units_latest":
            return ["id", "name"]
        if table == "teamlandkarte_v_teammeter_team_competences_latest":
            return ["ouid", "competence_id", "top_competency"]
        if table == "teamlandkarte_v_team_references_latest":
            return ["ouid", "partner_id", "projects"]
        return []

    def get_all_role_names(self) -> list[str]:
        return ["Backend Developer", "Frontend Developer", "Data Engineer"]

    def get_all_competence_names(self) -> list[str]:
        return ["python", "fastapi", "javascript", "spark"]

    def get_open_tasks(self, limit: int = 20):
        return []

    def get_all_capacities_with_competences(self):
        return []

    def get_recent_free_capacities(self, limit: int = 20):
        return []

    def get_all_teams(self) -> list[Team]:
        return list(self._teams)

    def get_team_by_id(self, team_id):  # pragma: no cover
        for t in self._teams:
            if t.team_id == str(team_id):
                return t
        return None


def _patch_capturing_store(
    monkeypatch: pytest.MonkeyPatch,
) -> list[dict[str, Any]]:
    """Capture every payload passed to ``SearchCache.store_search``."""
    captured: list[dict[str, Any]] = []
    original = SearchCache.store_search

    def _capturing(self, *, task_id, requirements, results):
        captured.append(results)
        return original(
            self,
            task_id=task_id,
            requirements=requirements,
            results=results,
        )

    monkeypatch.setattr(SearchCache, "store_search", _capturing)
    return captured


def _build_server_with_llm_mock(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path,
    *,
    team_responses: dict[str, Any],
) -> Any:
    """Build a server backed by ``_FakeDB`` and a mocked LLM.

    ``team_responses`` maps each ``team_id`` (str) to either a JSON
    string (success) or an :class:`AzureAPIError` instance (simulated
    failure). The fake ``chat_completion`` parses the team_id out of
    the prompt's ``ID: <team_id>`` line that
    :func:`_build_user_prompt_team_for_task` embeds, so the mapping is
    deterministic regardless of ``asyncio.gather`` scheduling.
    """
    cfg = tmp_path / "config.toml"
    cfg.write_text(_CONFIG_TEMPLATE, encoding="utf-8")

    fake_db = _FakeDB(_make_teams())
    monkeypatch.setattr(
        mcp_mod, "create_db_client", lambda *_a, **_k: fake_db
    )

    async def _fake_chat(self, system: str, user: str) -> str:  # noqa: ARG001
        team_id = ""
        for line in user.splitlines():
            if line.startswith("ID: "):
                team_id = line[len("ID: "):].strip()
                break
        outcome = team_responses.get(team_id)
        if isinstance(outcome, BaseException):
            raise outcome
        if outcome is None:
            # Defensive default: any unmapped team is reported as
            # ``Irrelevant`` so the test still terminates.
            return json.dumps(
                {"category": "Irrelevant", "rationale": "default"},
                ensure_ascii=False,
            )
        return outcome

    monkeypatch.setattr(
        "teamlandkarte_mcp.azure.openai_client."
        "AzureOpenAIClient.chat_completion",
        _fake_chat,
    )

    monkeypatch.setenv("DATA_LAKE_USERNAME", "u")
    monkeypatch.setenv("DATA_LAKE_PASSWORD", "p")
    monkeypatch.setenv("AZURE_OPENAI_LLM_API_KEY", "k")

    return build_server(str(cfg))


@pytest.mark.asyncio
async def test_e2e_find_matching_teams_llm_fulltext_full_path(
    monkeypatch: pytest.MonkeyPatch, tmp_path
) -> None:
    """End-to-end smoke test for ``find_matching_teams`` (llm_fulltext).

    Validates: requirements 7.1 (LLM-categorisation per team), 7.6
    (LLM error surfaces in ``errors`` list), 8.1, 8.2 (cache markers)
    and 8.4 (LLM-mode column layout).
    """
    # Two valid LLM payloads and one simulated failure.
    team_responses: dict[str, Any] = {
        "t1": json.dumps(
            {"category": "Top", "rationale": "Backend match."},
            ensure_ascii=False,
        ),
        "t2": json.dumps(
            {"category": "Partial", "rationale": "Frontend overlap."},
            ensure_ascii=False,
        ),
        "t3": AzureAPIError("LLM call failed"),
    }

    captured = _patch_capturing_store(monkeypatch)
    srv = _build_server_with_llm_mock(
        monkeypatch, tmp_path, team_responses=team_responses
    )

    # ------------------------------------------------------------------
    # 1) find_matching_teams (llm_fulltext mode)
    # ------------------------------------------------------------------
    out = await _call_tool(
        srv,
        "find_matching_teams",
        {
            "role_name": "Backend Developer",
            "competences": ["python"],
            "matching_method": "llm_fulltext",
        },
    )

    # SEARCH_ID + META markers (Anforderungen 8.1, 8.3).
    search_id = _extract_search_id(out)
    meta = _extract_meta(out)
    assert meta.get("search_id") == search_id
    assert meta.get("search_type") == "team_search"
    assert meta.get("matching_method") == "llm_fulltext"

    # The errored team (t3) must be visible in the rendered output via
    # the ``## Errors`` section (Anforderung 7.6).
    assert "## Errors" in out, out
    assert "t3" in out, out
    # Team t3 must NOT appear in the rendered category table because
    # errored teams never end up in by_category.
    # (We check the persisted payload below for the strict invariant.)

    # The team-search LLM column layout must be present
    # (Anforderung 8.4).
    for header in (
        "Team Name",
        "Schwerpunkt",
        "Top-Kompetenzen",
        "Category",
        "Begründung",
    ):
        assert header in out, (
            f"missing team-search llm-mode header {header!r} in:\n{out}"
        )

    # The persisted SearchCache payload carries the team_search markers
    # and the errors list (Anforderungen 7.6, 8.2).
    assert captured, "SearchCache.store_search was never called"
    payload = captured[-1]
    assert payload["search_type"] == "team_search"
    assert payload["matching_method"] == "llm_fulltext"

    errors = payload.get("errors") or []
    error_ids = {e["item_id"] for e in errors}
    assert error_ids == {"t3"}, errors

    by_category = payload.get("by_category") or {}
    flat_items = [item for items in by_category.values() for item in items]
    flat_ids = {str(it.get("team_id")) for it in flat_items}
    # The two successful teams are present, the errored team is not.
    assert flat_ids == {"t1", "t2"}, flat_ids
    # by_category must NOT contain the errored team.
    assert "t3" not in flat_ids

    # Each persisted item carries category + rationale fields.
    for item in flat_items:
        assert "team_id" in item
        assert "category" in item
        assert "rationale" in item
        assert isinstance(item["rationale"], str)
        assert item["rationale"].strip()

    # ------------------------------------------------------------------
    # 2) get_results_by_category — renders the LLM-mode team table.
    # ------------------------------------------------------------------
    default_category = str(meta.get("default_category") or "Top")
    cat_out = await _call_tool(
        srv,
        "get_results_by_category",
        {
            "search_id": search_id,
            "category": default_category,
            "page": 1,
            "page_size": 20,
        },
    )

    cat_meta = _extract_meta(cat_out)
    assert cat_meta.get("search_type") == "team_search"
    assert cat_meta.get("matching_method") == "llm_fulltext"
    assert cat_meta.get("search_id") == search_id

    for header in (
        "Team Name",
        "Schwerpunkt",
        "Top-Kompetenzen",
        "Category",
        "Begründung",
    ):
        assert header in cat_out, (
            f"missing team-search llm-mode header {header!r} in "
            f"get_results_by_category output:\n{cat_out}"
        )
    # Score-only columns must NOT appear in LLM-mode tables.
    assert "Role Score" not in cat_out
    assert "Overall Score" not in cat_out

    # ------------------------------------------------------------------
    # 3) filter_search_results — round-trip on the team_search entry.
    # ------------------------------------------------------------------
    filtered_out = await _call_tool(
        srv,
        "filter_search_results",
        {
            "search_id": search_id,
            "role_filter": "Backend",
        },
    )

    assert "FILTER_ID=" in filtered_out, filtered_out
    filter_meta = _extract_meta(filtered_out)
    assert filter_meta.get("search_type") == "team_search"
    assert filter_meta.get("matching_method") == "llm_fulltext"
    assert filter_meta.get("status") == "ok"
    # Only Team Alpha's focus_name matches "Backend"; t3 is in errors
    # (not in by_category), so the filtered total must be exactly 1.
    assert filter_meta.get("total") == 1, filter_meta

    assert "## Applied Filters" in filtered_out
    assert "role_filter" in filtered_out
    assert "Backend" in filtered_out
    # LLM-mode preview must include the Begründung column.
    assert "Begründung" in filtered_out
    assert "Filtered total results: 1" in filtered_out