# Bugfix: llm-competence-inference, Property 1: Fault Condition # Kompetenz-Inferenz liefert stets leere Liste (Bug-Bestätigung) """Property-based exploration test for the competence inference bug. This test encodes the EXPECTED (correct) behaviour: given a non-empty task text and a non-empty competence list in the DB, `infer_competences` should return a non-empty list of up to 10 (name, confidence) tuples where each name is in the DB competence list and confidence is in [0.0, 1.0]. On UNFIXED code this test is EXPECTED TO FAIL — the failure confirms the bug exists (infer_competences either doesn't exist or always returns []). **Validates: Requirements 1.1, 1.2, 1.3, 2.1, 2.2, 2.3** """ from __future__ import annotations import json import pytest from hypothesis import given, settings from hypothesis import strategies as st from teamlandkarte_mcp.matching.vocabulary import VocabularyCache # --------------------------------------------------------------------------- # Strategies # --------------------------------------------------------------------------- # Non-empty task text (printable characters, min 1 char) _task_text_strategy = st.text(min_size=1, max_size=200).filter(lambda s: s.strip()) # Non-empty competence list (each competence is a non-empty string) _competence_list_strategy = st.lists( st.text(min_size=1, max_size=50).filter(lambda s: s.strip()), min_size=1, max_size=50, unique=True, ) # --------------------------------------------------------------------------- # Fakes # --------------------------------------------------------------------------- class _FakeDB: """Stub DBClient that returns a configurable competence list.""" def __init__(self, competences: list[str]) -> None: self._competences = competences def get_all_competence_names(self) -> list[str]: return self._competences def get_all_role_names(self) -> list[str]: return ["Software Engineer"] class _FakeLLMClient: """Stub AzureOpenAIClient that returns valid JSON with competences. Simulates a working LLM that picks up to 5 competences from the provided list (via the user prompt) and assigns confidence scores. """ def __init__(self, competences: list[str]) -> None: self._competences = competences async def chat_completion(self, system: str, user: str) -> str: # Pick up to 5 competences from the list and return valid JSON selected = self._competences[:5] result = { "competences": [ {"name": name, "confidence": round(0.6 + i * 0.05, 2)} for i, name in enumerate(selected) ] } return json.dumps(result) # --------------------------------------------------------------------------- # Property Test # --------------------------------------------------------------------------- @settings(max_examples=100) @given( task_text=_task_text_strategy, competences=_competence_list_strategy, ) @pytest.mark.asyncio async def test_infer_competences_returns_nonempty_for_valid_input( task_text: str, competences: list[str], ) -> None: """Property 1 (Fault Condition): For any non-empty task text and non-empty competence list in the DB, infer_competences MUST return a non-empty list of up to 10 tuples (name, confidence) where each name is in the DB competence list and confidence is in [0.0, 1.0]. On unfixed code this WILL FAIL because: - The method `infer_competences` does not exist on VocabularyCache, OR - It always returns [] """ db = _FakeDB(competences=competences) client = _FakeLLMClient(competences=competences) cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type] # The method must exist and be callable assert hasattr(cache, "infer_competences"), ( "VocabularyCache has no 'infer_competences' method — " "the competence inference is not implemented" ) result = await cache.infer_competences(task_text=task_text) # Must return a non-empty list assert isinstance(result, list), ( f"Expected list, got {type(result).__name__}" ) assert len(result) > 0, ( f"infer_competences('{task_text[:50]}...') returned empty list [] " f"instead of competences — bug confirmed: no LLM inference happening" ) # Must return at most 10 tuples assert len(result) <= 10, ( f"Expected at most 10 competences, got {len(result)}" ) # Each element must be a tuple (name, confidence) for item in result: assert isinstance(item, tuple) and len(item) == 2, ( f"Expected tuple (name, confidence), got {item!r}" ) name, confidence = item assert name in competences, ( f"Returned competence '{name}' not in DB competence list" ) assert isinstance(confidence, float), ( f"Expected float confidence, got {type(confidence).__name__}" ) assert 0.0 <= confidence <= 1.0, ( f"Confidence {confidence} out of range [0.0, 1.0]" )