# Bugfix: llm-competence-inference, Property 2: Preservation # Rollen-Inferenz und Leer-Eingaben unverändert """Property-based preservation tests for the competence inference bugfix. These tests verify that EXISTING behaviour is preserved both before and after the fix: 1. `infer_primary_role` continues to return a valid (role, confidence) tuple for non-empty task texts (or None on error). 2. Empty task text yields [] from `infer_competences` (if the method exists). 3. Empty competence list in DB yields [] from `infer_competences` (if exists). On UNFIXED code these tests are EXPECTED TO PASS — they confirm baseline behaviour that must not regress after the fix is applied. **Validates: Requirements 3.1, 3.2, 3.3, 3.4, 3.5** """ from __future__ import annotations import json import pytest from hypothesis import given, settings from hypothesis import strategies as st from teamlandkarte_mcp.matching.vocabulary import VocabularyCache # --------------------------------------------------------------------------- # Strategies # --------------------------------------------------------------------------- # Non-empty task text (printable ASCII, min 1 char after strip) _nonempty_task_text = st.text( alphabet=st.characters( whitelist_categories=("L", "N", "Zs"), max_codepoint=127, ), min_size=1, max_size=150, ).filter(lambda s: s.strip()) # Empty or whitespace-only task text _empty_task_text = st.one_of( st.just(""), st.from_regex(r"^\s*$", fullmatch=True), ) # Non-empty competence list _competence_list = st.lists( st.text( alphabet=st.characters(whitelist_categories=("L",), max_codepoint=127), min_size=2, max_size=30, ).filter(lambda s: s.strip()), min_size=1, max_size=30, unique=True, ) # Non-empty role list _role_list = st.lists( st.text( alphabet=st.characters(whitelist_categories=("L",), max_codepoint=127), min_size=3, max_size=30, ).filter(lambda s: s.strip()), min_size=1, max_size=10, unique=True, ) # --------------------------------------------------------------------------- # Fakes # --------------------------------------------------------------------------- class _FakeDB: """Stub DBClient with configurable roles and competences.""" def __init__( self, roles: list[str] | None = None, competences: list[str] | None = None, ) -> None: self._roles = roles or ["Software Engineer"] self._competences = competences or [] def get_all_role_names(self) -> list[str]: return self._roles def get_all_competence_names(self) -> list[str]: return self._competences class _FakeRoleLLMClient: """Stub AzureOpenAIClient that returns a valid role from the DB list. Picks the first role from the list provided at construction time. """ def __init__(self, role: str, confidence: float = 0.85) -> None: self._role = role self._confidence = confidence async def chat_completion(self, system: str, user: str) -> str: return json.dumps({"role": self._role, "confidence": self._confidence}) class _FakeCompetenceLLMClient: """Stub AzureOpenAIClient that returns valid competence JSON.""" def __init__(self, competences: list[str] | None = None) -> None: self._competences = competences or [] async def chat_completion(self, system: str, user: str) -> str: selected = self._competences[:3] result = { "competences": [ {"name": name, "confidence": 0.8} for name in selected ] } return json.dumps(result) # --------------------------------------------------------------------------- # Property-Based Test 1: infer_primary_role preservation # --------------------------------------------------------------------------- @settings(max_examples=100) @given( task_text=_nonempty_task_text, roles=_role_list, role_index=st.data(), ) @pytest.mark.asyncio async def test_infer_primary_role_returns_valid_tuple_for_nonempty_text( task_text: str, roles: list[str], role_index: st.DataObject, ) -> None: """Property 2.1 (Preservation): For all non-empty task texts, `infer_primary_role` returns a tuple (role_name, confidence) where role_name is in the DB role list and confidence is in [0.0, 1.0], or None on failure. This verifies Requirement 3.1 and 3.5: infer_primary_role must continue to select exactly one role from the role list with a confidence score. **Validates: Requirements 3.1, 3.4, 3.5** """ idx = role_index.draw(st.integers(min_value=0, max_value=len(roles) - 1)) chosen_role = roles[idx] db = _FakeDB(roles=roles) client = _FakeRoleLLMClient(role=chosen_role, confidence=0.85) cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type] result = await cache.infer_primary_role(task_text=task_text) # Must return a valid tuple or None (None only on error/unknown role) assert result is not None, ( f"infer_primary_role returned None for valid role '{chosen_role}' " f"in list {roles}" ) role_name, confidence = result assert role_name in roles, ( f"Returned role '{role_name}' not in DB role list" ) assert isinstance(confidence, float), ( f"Expected float confidence, got {type(confidence).__name__}" ) assert 0.0 <= confidence <= 1.0, ( f"Confidence {confidence} out of range [0.0, 1.0]" ) # --------------------------------------------------------------------------- # Property-Based Test 2: Empty task text → infer_competences returns [] # --------------------------------------------------------------------------- @settings(max_examples=100) @given(task_text=_empty_task_text) @pytest.mark.asyncio async def test_infer_competences_returns_empty_for_empty_text( task_text: str, ) -> None: """Property 2.2 (Preservation): For all empty/whitespace-only task texts, `infer_competences` returns []. If the method does not exist yet (unfixed code), the test passes trivially because the preservation property is satisfied — there is no method that could produce incorrect results for empty input. **Validates: Requirements 3.2, 3.3** """ competences = ["Python", "Java", "TypeScript", "React", "Docker"] db = _FakeDB(roles=["Engineer"], competences=competences) client = _FakeCompetenceLLMClient(competences=competences) cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type] # If infer_competences doesn't exist yet, preservation is trivially satisfied if not hasattr(cache, "infer_competences"): return result = await cache.infer_competences(task_text=task_text) assert result == [], ( f"infer_competences('{task_text!r}') should return [] for " f"empty/whitespace text, got {result!r}" ) # --------------------------------------------------------------------------- # Property-Based Test 3: Empty competence list in DB → infer_competences [] # --------------------------------------------------------------------------- @settings(max_examples=100) @given(task_text=_nonempty_task_text) @pytest.mark.asyncio async def test_infer_competences_returns_empty_for_empty_db_competences( task_text: str, ) -> None: """Property 2.3 (Preservation): For an empty competence list in the DB, `infer_competences` returns [] regardless of task text. If the method does not exist yet (unfixed code), the test passes trivially because the preservation property is satisfied — there is no method that could produce incorrect results for empty DB competences. **Validates: Requirements 3.2, 3.3** """ db = _FakeDB(roles=["Engineer"], competences=[]) # Empty competence list client = _FakeCompetenceLLMClient(competences=[]) cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type] # If infer_competences doesn't exist yet, preservation is trivially satisfied if not hasattr(cache, "infer_competences"): return result = await cache.infer_competences(task_text=task_text) assert result == [], ( f"infer_competences('{task_text[:50]}...') should return [] when " f"DB has no competences, got {result!r}" )