Orchestrator/bahn/teamlandkarte-mcp/tests/test_competence_inference_fault_pbt.py

# Bugfix: llm-competence-inference, Property 1: Fault Condition
# Kompetenz-Inferenz liefert stets leere Liste (Bug-Bestätigung)

"""Property-based exploration test for the competence inference bug.

This test encodes the EXPECTED (correct) behaviour: given a non-empty task
text and a non-empty competence list in the DB, `infer_competences` should
return a non-empty list of up to 10 (name, confidence) tuples where each
name is in the DB competence list and confidence is in [0.0, 1.0].

On UNFIXED code this test is EXPECTED TO FAIL — the failure confirms the bug
exists (infer_competences either doesn't exist or always returns []).

**Validates: Requirements 1.1, 1.2, 1.3, 2.1, 2.2, 2.3**
"""
from __future__ import annotations

import json

import pytest
from hypothesis import given, settings
from hypothesis import strategies as st

from teamlandkarte_mcp.matching.vocabulary import VocabularyCache


# ---------------------------------------------------------------------------
# Strategies
# ---------------------------------------------------------------------------

# Non-empty task text (printable characters, min 1 char)
_task_text_strategy = st.text(min_size=1, max_size=200).filter(lambda s: s.strip())

# Non-empty competence list (each competence is a non-empty string)
_competence_list_strategy = st.lists(
    st.text(min_size=1, max_size=50).filter(lambda s: s.strip()),
    min_size=1,
    max_size=50,
    unique=True,
)


# ---------------------------------------------------------------------------
# Fakes
# ---------------------------------------------------------------------------


class _FakeDB:
    """Stub DBClient that returns a configurable competence list."""

    def __init__(self, competences: list[str]) -> None:
        self._competences = competences

    def get_all_competence_names(self) -> list[str]:
        return self._competences

    def get_all_role_names(self) -> list[str]:
        return ["Software Engineer"]


class _FakeLLMClient:
    """Stub AzureOpenAIClient that returns valid JSON with competences.

    Simulates a working LLM that picks up to 5 competences from the
    provided list (via the user prompt) and assigns confidence scores.
    """

    def __init__(self, competences: list[str]) -> None:
        self._competences = competences

    async def chat_completion(self, system: str, user: str) -> str:
        # Pick up to 5 competences from the list and return valid JSON
        selected = self._competences[:5]
        result = {
            "competences": [
                {"name": name, "confidence": round(0.6 + i * 0.05, 2)}
                for i, name in enumerate(selected)
            ]
        }
        return json.dumps(result)


# ---------------------------------------------------------------------------
# Property Test
# ---------------------------------------------------------------------------


@settings(max_examples=100)
@given(
    task_text=_task_text_strategy,
    competences=_competence_list_strategy,
)
@pytest.mark.asyncio
async def test_infer_competences_returns_nonempty_for_valid_input(
    task_text: str,
    competences: list[str],
) -> None:
    """Property 1 (Fault Condition): For any non-empty task text and non-empty
    competence list in the DB, infer_competences MUST return a non-empty list
    of up to 10 tuples (name, confidence) where each name is in the DB
    competence list and confidence is in [0.0, 1.0].

    On unfixed code this WILL FAIL because:
    - The method `infer_competences` does not exist on VocabularyCache, OR
    - It always returns []
    """
    db = _FakeDB(competences=competences)
    client = _FakeLLMClient(competences=competences)
    cache = VocabularyCache(db=db, client=client)  # type: ignore[arg-type]

    # The method must exist and be callable
    assert hasattr(cache, "infer_competences"), (
        "VocabularyCache has no 'infer_competences' method — "
        "the competence inference is not implemented"
    )

    result = await cache.infer_competences(task_text=task_text)

    # Must return a non-empty list
    assert isinstance(result, list), (
        f"Expected list, got {type(result).__name__}"
    )
    assert len(result) > 0, (
        f"infer_competences('{task_text[:50]}...') returned empty list [] "
        f"instead of competences — bug confirmed: no LLM inference happening"
    )

    # Must return at most 10 tuples
    assert len(result) <= 10, (
        f"Expected at most 10 competences, got {len(result)}"
    )

    # Each element must be a tuple (name, confidence)
    for item in result:
        assert isinstance(item, tuple) and len(item) == 2, (
            f"Expected tuple (name, confidence), got {item!r}"
        )
        name, confidence = item
        assert name in competences, (
            f"Returned competence '{name}' not in DB competence list"
        )
        assert isinstance(confidence, float), (
            f"Expected float confidence, got {type(confidence).__name__}"
        )
        assert 0.0 <= confidence <= 1.0, (
            f"Confidence {confidence} out of range [0.0, 1.0]"
        )