Orchestrator/bahn/teamlandkarte-mcp/tests/test_meta_matching_method_pbt.py

# Feature: llm-fulltext-matching, Property 11: META enthält das verwendete Verfahren
"""Property test verifying that META-JSON includes ``matching_method``.

The MCP server emits a ``META=...`` line in the response of
``find_matching_capacities`` and ``find_matching_tasks`` containing a
JSON object with ``search_id``, ``filter_id``, ``default_category`` and
``matching_method``. For any successful tool call, the value of
``matching_method`` in this JSON must be exactly the method that was
used (``"score"`` or ``"llm_fulltext"``).

Hypothesis would only enumerate two distinct values here, so the
property is encoded as a parametrized pytest test which still rebuilds
the server per case for full isolation. The header retains the
"Property 11" tag for traceability with the spec.

**Validates: Requirements 1.6, 9.5**
"""

from __future__ import annotations

import json
from datetime import date
from typing import Any
from unittest.mock import AsyncMock

import pytest

import teamlandkarte_mcp.mcp_server as mcp_mod
from teamlandkarte_mcp.mcp_server import build_server
from teamlandkarte_mcp.models import Capacity


_CONFIG_TEMPLATE = """
[database]
host='x'
port=1
username='u'
password='p'
backend='trino'

[matching]
competence_weight=0.8
role_weight=0.2
require_confirmation=false

[cache]
db_ttl_hours=1
search_ttl_minutes=60
max_size=100

[azure_openai]
endpoint='https://example.openai.azure.com'
api_version='2024-02-15-preview'
chat_deployment='gpt-4'
""".strip()


class _FakeDB:
    """Minimal fake DB that returns one matching capacity."""

    def test_connection(self) -> None:
        return

    def get_table_columns(self, table: str) -> list[str]:
        if table == "teamlandkarte_v_capacity_roles_latest":
            return ["name", "active", "staffing_board_relevant"]
        if table == "teamlandkarte_v_capacities_latest":
            return ["creation_date"]
        if table == "teamlandkarte_v_teams_latest":
            return [
                "team_id",
                "ouid",
                "about_us",
                "offerings",
                "interests",
                "focus_name",
            ]
        if table == "teamlandkarte_v_teammeter_organizational_units_latest":
            return ["id", "name"]
        if table == "teamlandkarte_v_teammeter_team_competences_latest":
            return ["ouid", "competence_id", "top_competency"]
        if table == "teamlandkarte_v_team_references_latest":
            return ["ouid", "partner_id", "projects"]
        return []

    def get_all_role_names(self) -> list[str]:
        return ["X"]

    def get_all_competence_names(self) -> list[str]:
        return ["A"]

    def get_open_tasks(self, limit: int = 20):
        return []

    def get_all_capacities_with_competences(self):
        return [
            Capacity(
                id=1,
                owner_name="A",
                role_name="X",
                role_level=None,
                begin_date=date(2025, 1, 1),
                end_date=date(2025, 12, 31),
                competences=["A"],
            )
        ]

    def get_recent_free_capacities(self, limit: int = 20):
        return []

    # LLM-fulltext path needs the batch description/cert/ref methods.
    def batch_get_capacity_descriptions(
        self, capacity_ids: list[Any]
    ) -> dict[str, str | None]:
        return {str(i): None for i in capacity_ids}

    def batch_get_capacity_certificates(
        self, capacity_ids: list[Any]
    ) -> dict[str, list[str]]:
        return {str(i): [] for i in capacity_ids}

    def batch_get_capacity_references(
        self, capacity_ids: list[Any]
    ) -> dict[str, list[dict]]:
        return {str(i): [] for i in capacity_ids}


def _result_to_text(result: Any) -> str:
    if isinstance(result, tuple) and len(result) == 2:
        content, structured = result
        if isinstance(structured, dict) and isinstance(
            structured.get("result"), str
        ):
            return structured["result"]
        if isinstance(content, list) and content:
            text = getattr(content[0], "text", None)
            if isinstance(text, str):
                return text
    return str(result)


async def _call_tool(srv: Any, name: str, args: dict[str, Any]) -> str:
    return _result_to_text(await srv.call_tool(name, args))


def _extract_meta(output: str) -> dict[str, Any]:
    for line in output.splitlines():
        if line.startswith("META="):
            return json.loads(line[len("META=") :])
    raise AssertionError(f"no META= line in tool output:\n{output}")


@pytest.fixture
def _env(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("DATA_LAKE_USERNAME", "u")
    monkeypatch.setenv("DATA_LAKE_PASSWORD", "p")
    monkeypatch.setenv("AZURE_OPENAI_LLM_API_KEY", "k")


@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["score", "llm_fulltext"])
async def test_meta_includes_matching_method(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path,
    _env,
    method: str,
) -> None:
    """Property 11: META carries the matching_method actually used."""
    cfg = tmp_path / "config.toml"
    cfg.write_text(_CONFIG_TEMPLATE, encoding="utf-8")

    monkeypatch.setattr(
        mcp_mod, "create_db_client", lambda *_a, **_k: _FakeDB()
    )

    # A single payload that satisfies both consumers:
    #   - score path reads ``similarity``
    #   - llm_fulltext path reads ``category``/``rationale``
    monkeypatch.setattr(
        "teamlandkarte_mcp.azure.openai_client.AzureOpenAIClient.chat_completion",
        AsyncMock(
            return_value=(
                '{"similarity": 1.0, "category": "Top", "rationale": "ok"}'
            )
        ),
    )

    srv = build_server(str(cfg))

    res = await _call_tool(
        srv,
        "find_matching_capacities",
        {
            "role_name": "X",
            "competences": ["A"],
            "matching_method": method,
        },
    )

    meta = _extract_meta(res)
    assert meta.get("matching_method") == method, (
        f"expected matching_method={method!r}, got META={meta!r}"
    )