Migrate all repos into monorepo context folders
Bahn: aisupport, Analyse-O2C-C2S, awesome-bahn-mcp-servers, beam-mcp,
Confluence_Bot, db-planet-mcp-server, O2C-Harness, project-audit,
Projekt-KIQ-HP, teamlandkarte-mcp
Dhive: Jury-Voting
Privat: CV, NoteGraph (NOTE: NoteGraph needs complete redo after consolidation)
Shared: AI-Orchestrator, OrgMyLife, power_skills_and_more
Shared/references: symphony (read-only)
Bahn repos remain available as independent remotes - this monorepo
pulls them in via subtree, the originals are untouched.
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
# Feature: remove-embedding-competence-similarity, Property 4: Graceful Degradation bei LLM-Fehler
|
||||
"""Property-based tests for graceful degradation on LLM failure."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from teamlandkarte_mcp.matching.similarity import SimilarityEngine
|
||||
from teamlandkarte_mcp.matching.vocabulary import VocabularyCache
|
||||
|
||||
|
||||
# Strategy: role name strings (ASCII letters only, 1-50 chars,
|
||||
# no "(unknown)")
|
||||
_role_str = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L",),
|
||||
max_codepoint=127,
|
||||
),
|
||||
min_size=1,
|
||||
max_size=50,
|
||||
).filter(lambda s: s.strip().lower() != "(unknown)")
|
||||
|
||||
# Strategy: non-empty task text (ASCII letters + spaces, 1-100 chars)
|
||||
_task_text = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "Zs"),
|
||||
max_codepoint=127,
|
||||
),
|
||||
min_size=1,
|
||||
max_size=100,
|
||||
).filter(lambda s: s.strip())
|
||||
|
||||
# Strategy: non-empty role names for DB list (ASCII letters, 3-30 chars)
|
||||
_role_name = st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L",),
|
||||
max_codepoint=127,
|
||||
),
|
||||
min_size=3,
|
||||
max_size=30,
|
||||
)
|
||||
|
||||
# Strategy: list of unique role names (1-10 roles)
|
||||
_role_list = st.lists(_role_name, min_size=1, max_size=10, unique=True)
|
||||
|
||||
# Strategy: exception class to raise
|
||||
_exception_class = st.sampled_from([TimeoutError, RuntimeError, ValueError])
|
||||
|
||||
|
||||
class _FailingClient:
|
||||
"""Stub AzureOpenAIClient that raises a configured exception."""
|
||||
|
||||
def __init__(self, exc_class: type) -> None:
|
||||
self._exc_class = exc_class
|
||||
|
||||
async def chat_completion(self, system: str, user: str) -> str:
|
||||
if self._exc_class is ValueError:
|
||||
raise ValueError("simulated LLM failure")
|
||||
elif self._exc_class is json.JSONDecodeError:
|
||||
raise json.JSONDecodeError("simulated", "", 0)
|
||||
else:
|
||||
raise self._exc_class("simulated LLM failure")
|
||||
|
||||
|
||||
class _JSONDecodeErrorClient:
|
||||
"""Stub that always raises json.JSONDecodeError."""
|
||||
|
||||
async def chat_completion(self, system: str, user: str) -> str:
|
||||
raise json.JSONDecodeError("simulated parse error", "", 0)
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
"""Stub DBClient that returns a configurable role list."""
|
||||
|
||||
def __init__(self, roles: list[str]) -> None:
|
||||
self._roles = roles
|
||||
|
||||
def get_all_role_names(self) -> list[str]:
|
||||
return self._roles
|
||||
|
||||
|
||||
# **Validates: Requirements 5.6**
|
||||
@settings(max_examples=100)
|
||||
@given(role_a=_role_str, role_b=_role_str, exc_class=_exception_class)
|
||||
@pytest.mark.asyncio
|
||||
async def test_compute_role_similarity_returns_zero_on_exception(
|
||||
role_a: str, role_b: str, exc_class: type
|
||||
) -> None:
|
||||
"""Property 4a: compute_role_similarity returns 0.0 on any LLM exception.
|
||||
|
||||
For any two valid role names and any exception type raised by the LLM,
|
||||
compute_role_similarity shall return 0.0 without propagating the
|
||||
exception to the caller.
|
||||
"""
|
||||
# Skip identical roles (they short-circuit to 1.0 without LLM call)
|
||||
if role_a.strip().lower() == role_b.strip().lower():
|
||||
return
|
||||
|
||||
client = _FailingClient(exc_class=exc_class)
|
||||
engine = SimilarityEngine(client=client) # type: ignore[arg-type]
|
||||
|
||||
result = await engine.compute_role_similarity(role_a, role_b)
|
||||
|
||||
assert result == 0.0, (
|
||||
f"Expected 0.0 on {exc_class.__name__}, got {result} "
|
||||
f"(roles: {role_a!r}, {role_b!r})"
|
||||
)
|
||||
|
||||
|
||||
# **Validates: Requirements 5.6**
|
||||
@settings(max_examples=100)
|
||||
@given(role_a=_role_str, role_b=_role_str)
|
||||
@pytest.mark.asyncio
|
||||
async def test_compute_role_similarity_returns_zero_on_json_decode_error(
|
||||
role_a: str, role_b: str,
|
||||
) -> None:
|
||||
"""Property 4b: compute_role_similarity returns 0.0 on JSONDecodeError.
|
||||
|
||||
json.JSONDecodeError requires positional args, so it is tested
|
||||
separately with a dedicated stub.
|
||||
"""
|
||||
# Skip identical roles (they short-circuit to 1.0 without LLM call)
|
||||
if role_a.strip().lower() == role_b.strip().lower():
|
||||
return
|
||||
|
||||
client = _JSONDecodeErrorClient()
|
||||
engine = SimilarityEngine(client=client) # type: ignore[arg-type]
|
||||
|
||||
result = await engine.compute_role_similarity(role_a, role_b)
|
||||
|
||||
assert result == 0.0, (
|
||||
f"Expected 0.0 on JSONDecodeError, got {result} "
|
||||
f"(roles: {role_a!r}, {role_b!r})"
|
||||
)
|
||||
|
||||
|
||||
# **Validates: Requirements 3.5**
|
||||
@settings(max_examples=100)
|
||||
@given(task_text=_task_text, roles=_role_list, exc_class=_exception_class)
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_primary_role_returns_none_on_exception(
|
||||
task_text: str, roles: list[str], exc_class: type
|
||||
) -> None:
|
||||
"""Property 4c: infer_primary_role returns None on any LLM exception.
|
||||
|
||||
For any valid task text and role list, if the LLM raises any exception,
|
||||
infer_primary_role shall return None without propagating the exception
|
||||
to the caller.
|
||||
"""
|
||||
client = _FailingClient(exc_class=exc_class)
|
||||
db = _FakeDB(roles=roles)
|
||||
cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type]
|
||||
|
||||
result = await cache.infer_primary_role(task_text=task_text)
|
||||
|
||||
assert result is None, (
|
||||
f"Expected None on {exc_class.__name__}, got {result} "
|
||||
f"(task_text: {task_text!r})"
|
||||
)
|
||||
|
||||
|
||||
# **Validates: Requirements 3.5**
|
||||
@settings(max_examples=100)
|
||||
@given(task_text=_task_text, roles=_role_list)
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_primary_role_returns_none_on_json_decode_error(
|
||||
task_text: str, roles: list[str],
|
||||
) -> None:
|
||||
"""Property 4d: infer_primary_role returns None on JSONDecodeError.
|
||||
|
||||
json.JSONDecodeError requires positional args, so it is tested
|
||||
separately with a dedicated stub.
|
||||
"""
|
||||
client = _JSONDecodeErrorClient()
|
||||
db = _FakeDB(roles=roles)
|
||||
cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type]
|
||||
|
||||
result = await cache.infer_primary_role(task_text=task_text)
|
||||
|
||||
assert result is None, (
|
||||
f"Expected None on JSONDecodeError, got {result} "
|
||||
f"(task_text: {task_text!r})"
|
||||
)
|
||||
Reference in New Issue
Block a user