# Feature: remove-embedding-competence-similarity, Property 4: Graceful Degradation bei LLM-Fehler """Property-based tests for graceful degradation on LLM failure.""" from __future__ import annotations import json import pytest from hypothesis import given, settings from hypothesis import strategies as st from teamlandkarte_mcp.matching.similarity import SimilarityEngine from teamlandkarte_mcp.matching.vocabulary import VocabularyCache # Strategy: role name strings (ASCII letters only, 1-50 chars, # no "(unknown)") _role_str = st.text( alphabet=st.characters( whitelist_categories=("L",), max_codepoint=127, ), min_size=1, max_size=50, ).filter(lambda s: s.strip().lower() != "(unknown)") # Strategy: non-empty task text (ASCII letters + spaces, 1-100 chars) _task_text = st.text( alphabet=st.characters( whitelist_categories=("L", "Zs"), max_codepoint=127, ), min_size=1, max_size=100, ).filter(lambda s: s.strip()) # Strategy: non-empty role names for DB list (ASCII letters, 3-30 chars) _role_name = st.text( alphabet=st.characters( whitelist_categories=("L",), max_codepoint=127, ), min_size=3, max_size=30, ) # Strategy: list of unique role names (1-10 roles) _role_list = st.lists(_role_name, min_size=1, max_size=10, unique=True) # Strategy: exception class to raise _exception_class = st.sampled_from([TimeoutError, RuntimeError, ValueError]) class _FailingClient: """Stub AzureOpenAIClient that raises a configured exception.""" def __init__(self, exc_class: type) -> None: self._exc_class = exc_class async def chat_completion(self, system: str, user: str) -> str: if self._exc_class is ValueError: raise ValueError("simulated LLM failure") elif self._exc_class is json.JSONDecodeError: raise json.JSONDecodeError("simulated", "", 0) else: raise self._exc_class("simulated LLM failure") class _JSONDecodeErrorClient: """Stub that always raises json.JSONDecodeError.""" async def chat_completion(self, system: str, user: str) -> str: raise json.JSONDecodeError("simulated parse error", "", 0) class _FakeDB: """Stub DBClient that returns a configurable role list.""" def __init__(self, roles: list[str]) -> None: self._roles = roles def get_all_role_names(self) -> list[str]: return self._roles # **Validates: Requirements 5.6** @settings(max_examples=100) @given(role_a=_role_str, role_b=_role_str, exc_class=_exception_class) @pytest.mark.asyncio async def test_compute_role_similarity_returns_zero_on_exception( role_a: str, role_b: str, exc_class: type ) -> None: """Property 4a: compute_role_similarity returns 0.0 on any LLM exception. For any two valid role names and any exception type raised by the LLM, compute_role_similarity shall return 0.0 without propagating the exception to the caller. """ # Skip identical roles (they short-circuit to 1.0 without LLM call) if role_a.strip().lower() == role_b.strip().lower(): return client = _FailingClient(exc_class=exc_class) engine = SimilarityEngine(client=client) # type: ignore[arg-type] result = await engine.compute_role_similarity(role_a, role_b) assert result == 0.0, ( f"Expected 0.0 on {exc_class.__name__}, got {result} " f"(roles: {role_a!r}, {role_b!r})" ) # **Validates: Requirements 5.6** @settings(max_examples=100) @given(role_a=_role_str, role_b=_role_str) @pytest.mark.asyncio async def test_compute_role_similarity_returns_zero_on_json_decode_error( role_a: str, role_b: str, ) -> None: """Property 4b: compute_role_similarity returns 0.0 on JSONDecodeError. json.JSONDecodeError requires positional args, so it is tested separately with a dedicated stub. """ # Skip identical roles (they short-circuit to 1.0 without LLM call) if role_a.strip().lower() == role_b.strip().lower(): return client = _JSONDecodeErrorClient() engine = SimilarityEngine(client=client) # type: ignore[arg-type] result = await engine.compute_role_similarity(role_a, role_b) assert result == 0.0, ( f"Expected 0.0 on JSONDecodeError, got {result} " f"(roles: {role_a!r}, {role_b!r})" ) # **Validates: Requirements 3.5** @settings(max_examples=100) @given(task_text=_task_text, roles=_role_list, exc_class=_exception_class) @pytest.mark.asyncio async def test_infer_primary_role_returns_none_on_exception( task_text: str, roles: list[str], exc_class: type ) -> None: """Property 4c: infer_primary_role returns None on any LLM exception. For any valid task text and role list, if the LLM raises any exception, infer_primary_role shall return None without propagating the exception to the caller. """ client = _FailingClient(exc_class=exc_class) db = _FakeDB(roles=roles) cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type] result = await cache.infer_primary_role(task_text=task_text) assert result is None, ( f"Expected None on {exc_class.__name__}, got {result} " f"(task_text: {task_text!r})" ) # **Validates: Requirements 3.5** @settings(max_examples=100) @given(task_text=_task_text, roles=_role_list) @pytest.mark.asyncio async def test_infer_primary_role_returns_none_on_json_decode_error( task_text: str, roles: list[str], ) -> None: """Property 4d: infer_primary_role returns None on JSONDecodeError. json.JSONDecodeError requires positional args, so it is tested separately with a dedicated stub. """ client = _JSONDecodeErrorClient() db = _FakeDB(roles=roles) cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type] result = await cache.infer_primary_role(task_text=task_text) assert result is None, ( f"Expected None on JSONDecodeError, got {result} " f"(task_text: {task_text!r})" )