# Feature: remove-embedding-competence-similarity, Property 3: infer_primary_role Ausgabe-Validität """Property-based tests for infer_primary_role output validity.""" from __future__ import annotations import json import pytest from hypothesis import given, settings from hypothesis import strategies as st from teamlandkarte_mcp.matching.vocabulary import VocabularyCache # Strategy: non-empty task text (ASCII letters + spaces, 1-100 chars) _task_text = st.text( alphabet=st.characters( whitelist_categories=("L", "Zs"), max_codepoint=127, ), min_size=1, max_size=100, ).filter(lambda s: s.strip()) # Strategy: non-empty role names (ASCII letters, 3-30 chars) _role_name = st.text( alphabet=st.characters( whitelist_categories=("L",), max_codepoint=127, ), min_size=3, max_size=30, ) # Strategy: list of unique role names (1-10 roles) _role_list = st.lists(_role_name, min_size=1, max_size=10, unique=True) # Strategy: confidence score returned by the LLM (includes out-of-range # values to verify clamping) _llm_confidence = st.floats(min_value=-1.0, max_value=2.0) class _FakeDB: """Stub DBClient that returns a configurable role list.""" def __init__(self, roles: list[str]) -> None: self._roles = roles def get_all_role_names(self) -> list[str]: return self._roles class _FakeClient: """Stub AzureOpenAIClient that returns a chosen role and confidence.""" def __init__(self, role: str, confidence: float) -> None: self._role = role self._confidence = confidence async def chat_completion(self, system: str, user: str) -> str: return json.dumps({"role": self._role, "confidence": self._confidence}) # **Validates: Requirements 3.1, 3.4** @settings(max_examples=100) @given( task_text=_task_text, roles=_role_list, confidence=_llm_confidence, role_index=st.data(), ) @pytest.mark.asyncio async def test_infer_primary_role_output_in_role_list( task_text: str, roles: list[str], confidence: float, role_index: st.DataObject, ) -> None: """Property 3a: If infer_primary_role returns non-None, the role is in the DB list and confidence is in [0.0, 1.0]. For any non-empty task text and non-empty role list, if the LLM returns a valid role from the list, the output must satisfy: - role_name is an element of the database role list - confidence is a float in [0.0, 1.0] """ # Pick a random role from the generated list for the LLM to "return" idx = role_index.draw(st.integers(min_value=0, max_value=len(roles) - 1)) chosen_role = roles[idx] db = _FakeDB(roles=roles) client = _FakeClient(role=chosen_role, confidence=confidence) cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type] result = await cache.infer_primary_role(task_text=task_text) assert result is not None, ( f"Expected non-None result for valid role {chosen_role!r} in list" ) role_name, conf = result assert role_name in roles, ( f"Returned role {role_name!r} not in DB role list {roles!r}" ) assert isinstance(conf, float), ( f"Expected float confidence, got {type(conf)}" ) assert 0.0 <= conf <= 1.0, ( f"Confidence {conf} out of range [0.0, 1.0] " f"(llm_confidence: {confidence})" ) # **Validates: Requirement 3.4 (role must be in DB list)** @settings(max_examples=100) @given( task_text=_task_text, roles=_role_list, ) @pytest.mark.asyncio async def test_infer_primary_role_rejects_unknown_role( task_text: str, roles: list[str], ) -> None: """Property 3b: If the LLM returns a role NOT in the DB list, result is None. The VocabularyCache must validate that the returned role exists in the database role list. If it doesn't, the function returns None. """ # Return a role that is guaranteed not to be in the list fake_role = "ZZZZZ_NOT_A_REAL_ROLE_99999" assert fake_role not in roles db = _FakeDB(roles=roles) client = _FakeClient(role=fake_role, confidence=0.9) cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type] result = await cache.infer_primary_role(task_text=task_text) assert result is None, ( f"Expected None when LLM returns unknown role {fake_role!r}, " f"got {result!r}" )