Migrate all repos into monorepo context folders
Bahn: aisupport, Analyse-O2C-C2S, awesome-bahn-mcp-servers, beam-mcp,
Confluence_Bot, db-planet-mcp-server, O2C-Harness, project-audit,
Projekt-KIQ-HP, teamlandkarte-mcp
Dhive: Jury-Voting
Privat: CV, NoteGraph (NOTE: NoteGraph needs complete redo after consolidation)
Shared: AI-Orchestrator, OrgMyLife, power_skills_and_more
Shared/references: symphony (read-only)
Bahn repos remain available as independent remotes - this monorepo
pulls them in via subtree, the originals are untouched.
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
# Bugfix: llm-competence-inference, Property 1: Fault Condition
|
||||
# Kompetenz-Inferenz liefert stets leere Liste (Bug-Bestätigung)
|
||||
|
||||
"""Property-based exploration test for the competence inference bug.
|
||||
|
||||
This test encodes the EXPECTED (correct) behaviour: given a non-empty task
|
||||
text and a non-empty competence list in the DB, `infer_competences` should
|
||||
return a non-empty list of up to 10 (name, confidence) tuples where each
|
||||
name is in the DB competence list and confidence is in [0.0, 1.0].
|
||||
|
||||
On UNFIXED code this test is EXPECTED TO FAIL — the failure confirms the bug
|
||||
exists (infer_competences either doesn't exist or always returns []).
|
||||
|
||||
**Validates: Requirements 1.1, 1.2, 1.3, 2.1, 2.2, 2.3**
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from teamlandkarte_mcp.matching.vocabulary import VocabularyCache
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Non-empty task text (printable characters, min 1 char)
|
||||
_task_text_strategy = st.text(min_size=1, max_size=200).filter(lambda s: s.strip())
|
||||
|
||||
# Non-empty competence list (each competence is a non-empty string)
|
||||
_competence_list_strategy = st.lists(
|
||||
st.text(min_size=1, max_size=50).filter(lambda s: s.strip()),
|
||||
min_size=1,
|
||||
max_size=50,
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fakes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
"""Stub DBClient that returns a configurable competence list."""
|
||||
|
||||
def __init__(self, competences: list[str]) -> None:
|
||||
self._competences = competences
|
||||
|
||||
def get_all_competence_names(self) -> list[str]:
|
||||
return self._competences
|
||||
|
||||
def get_all_role_names(self) -> list[str]:
|
||||
return ["Software Engineer"]
|
||||
|
||||
|
||||
class _FakeLLMClient:
|
||||
"""Stub AzureOpenAIClient that returns valid JSON with competences.
|
||||
|
||||
Simulates a working LLM that picks up to 5 competences from the
|
||||
provided list (via the user prompt) and assigns confidence scores.
|
||||
"""
|
||||
|
||||
def __init__(self, competences: list[str]) -> None:
|
||||
self._competences = competences
|
||||
|
||||
async def chat_completion(self, system: str, user: str) -> str:
|
||||
# Pick up to 5 competences from the list and return valid JSON
|
||||
selected = self._competences[:5]
|
||||
result = {
|
||||
"competences": [
|
||||
{"name": name, "confidence": round(0.6 + i * 0.05, 2)}
|
||||
for i, name in enumerate(selected)
|
||||
]
|
||||
}
|
||||
return json.dumps(result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@settings(max_examples=100)
|
||||
@given(
|
||||
task_text=_task_text_strategy,
|
||||
competences=_competence_list_strategy,
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_competences_returns_nonempty_for_valid_input(
|
||||
task_text: str,
|
||||
competences: list[str],
|
||||
) -> None:
|
||||
"""Property 1 (Fault Condition): For any non-empty task text and non-empty
|
||||
competence list in the DB, infer_competences MUST return a non-empty list
|
||||
of up to 10 tuples (name, confidence) where each name is in the DB
|
||||
competence list and confidence is in [0.0, 1.0].
|
||||
|
||||
On unfixed code this WILL FAIL because:
|
||||
- The method `infer_competences` does not exist on VocabularyCache, OR
|
||||
- It always returns []
|
||||
"""
|
||||
db = _FakeDB(competences=competences)
|
||||
client = _FakeLLMClient(competences=competences)
|
||||
cache = VocabularyCache(db=db, client=client) # type: ignore[arg-type]
|
||||
|
||||
# The method must exist and be callable
|
||||
assert hasattr(cache, "infer_competences"), (
|
||||
"VocabularyCache has no 'infer_competences' method — "
|
||||
"the competence inference is not implemented"
|
||||
)
|
||||
|
||||
result = await cache.infer_competences(task_text=task_text)
|
||||
|
||||
# Must return a non-empty list
|
||||
assert isinstance(result, list), (
|
||||
f"Expected list, got {type(result).__name__}"
|
||||
)
|
||||
assert len(result) > 0, (
|
||||
f"infer_competences('{task_text[:50]}...') returned empty list [] "
|
||||
f"instead of competences — bug confirmed: no LLM inference happening"
|
||||
)
|
||||
|
||||
# Must return at most 10 tuples
|
||||
assert len(result) <= 10, (
|
||||
f"Expected at most 10 competences, got {len(result)}"
|
||||
)
|
||||
|
||||
# Each element must be a tuple (name, confidence)
|
||||
for item in result:
|
||||
assert isinstance(item, tuple) and len(item) == 2, (
|
||||
f"Expected tuple (name, confidence), got {item!r}"
|
||||
)
|
||||
name, confidence = item
|
||||
assert name in competences, (
|
||||
f"Returned competence '{name}' not in DB competence list"
|
||||
)
|
||||
assert isinstance(confidence, float), (
|
||||
f"Expected float confidence, got {type(confidence).__name__}"
|
||||
)
|
||||
assert 0.0 <= confidence <= 1.0, (
|
||||
f"Confidence {confidence} out of range [0.0, 1.0]"
|
||||
)
|
||||
Reference in New Issue
Block a user