ankn
a5f8fb49ab
Bahn: aisupport, Analyse-O2C-C2S, awesome-bahn-mcp-servers, beam-mcp,
Confluence_Bot, db-planet-mcp-server, O2C-Harness, project-audit,
Projekt-KIQ-HP, teamlandkarte-mcp
Dhive: Jury-Voting
Privat: CV, NoteGraph (NOTE: NoteGraph needs complete redo after consolidation)
Shared: AI-Orchestrator, OrgMyLife, power_skills_and_more
Shared/references: symphony (read-only)
Bahn repos remain available as independent remotes - this monorepo
pulls them in via subtree, the originals are untouched.
338 lines
12 KiB
Python
338 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
from typing import Any, cast
|
|
|
|
from teamlandkarte_mcp.matching.similarity import (
|
|
SimilarityEngine,
|
|
)
|
|
from teamlandkarte_mcp.matching.bm25 import Bm25Index
|
|
|
|
|
|
class _DummyClient: # pragma: no cover
|
|
"""Minimal stub for AzureOpenAIClient."""
|
|
|
|
async def chat_completion(self, system: str, user: str) -> str:
|
|
return json.dumps({"similarity": 0.5})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BM25 + RRF mode (now the only competence similarity path)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _bm25_engine(**kwargs: Any) -> SimilarityEngine:
|
|
"""Create a SimilarityEngine (BM25 is now always active)."""
|
|
return SimilarityEngine(
|
|
client=_DummyClient(), # type: ignore[arg-type]
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_exact_match_scores_one() -> None:
|
|
"""Required 'Python', candidate ['Python'] → score 1.0 with global index.
|
|
|
|
A global index over five distinct skills ensures 'python' has a positive
|
|
IDF, so the exact-match document receives a positive BM25 score and RRF
|
|
normalizes the single-result list to 1.0.
|
|
"""
|
|
engine = _bm25_engine()
|
|
global_index = Bm25Index(corpus=["Python", "JavaScript", "Java", "Go", "Ruby"])
|
|
result = await engine.compute_competence_similarity(
|
|
required=["Python"],
|
|
candidate=["Python"],
|
|
global_index=global_index,
|
|
)
|
|
assert result["Python"]["score"] == pytest.approx(1.0)
|
|
assert result["Python"]["best_match"] == "Python"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_no_overlap_scores_zero() -> None:
|
|
"""Required 'Python', candidate ['JavaScript'] → score 0.0."""
|
|
engine = _bm25_engine()
|
|
result = await engine.compute_competence_similarity(
|
|
required=["Python"],
|
|
candidate=["JavaScript"],
|
|
)
|
|
assert result["Python"]["score"] == 0.0
|
|
assert result["Python"]["best_match"] is None
|
|
assert "no token overlap" in str(result["Python"]["rationale"]).lower()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_multiple_required_no_overlap_all_zero() -> None:
|
|
"""Required ['Python', 'ML'], candidate JS-only → both scores 0.0."""
|
|
engine = _bm25_engine()
|
|
result = await engine.compute_competence_similarity(
|
|
required=["Python", "Machine Learning"],
|
|
candidate=["JavaScript", "TypeScript"],
|
|
)
|
|
assert result["Python"]["score"] == 0.0
|
|
assert result["Machine Learning"]["score"] == 0.0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_multiple_required_all_match() -> None:
|
|
"""Required ['Python', 'ML'], candidate exact match → both scores 1.0.
|
|
|
|
A global index over five distinct skills ensures all query tokens have
|
|
positive IDF, so both exact-match documents score 1.0 after RRF.
|
|
"""
|
|
engine = _bm25_engine()
|
|
global_index = Bm25Index(
|
|
corpus=["Python", "Machine Learning", "JavaScript", "TypeScript", "Java"]
|
|
)
|
|
result = await engine.compute_competence_similarity(
|
|
required=["Python", "Machine Learning"],
|
|
candidate=["Python", "Machine Learning"],
|
|
global_index=global_index,
|
|
)
|
|
assert result["Python"]["score"] == pytest.approx(1.0)
|
|
assert result["Machine Learning"]["score"] == pytest.approx(1.0)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_no_candidates_returns_zero() -> None:
|
|
"""Empty candidate list → score 0.0 for all required."""
|
|
engine = _bm25_engine()
|
|
result = await engine.compute_competence_similarity(
|
|
required=["Python"],
|
|
candidate=[],
|
|
)
|
|
assert result["Python"]["score"] == 0.0
|
|
assert result["Python"]["best_match"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_rationale_contains_bm25_rrf_on_match() -> None:
|
|
"""Rationale mentions 'BM25+RRF' when a match is found.
|
|
|
|
Uses a global index so the matched document receives a positive BM25
|
|
score and RRF fusion produces a non-zero result with the expected
|
|
rationale prefix.
|
|
"""
|
|
engine = _bm25_engine()
|
|
global_index = Bm25Index(
|
|
corpus=["Python", "Java", "JavaScript", "Go", "Ruby"]
|
|
)
|
|
result = await engine.compute_competence_similarity(
|
|
required=["Python"],
|
|
candidate=["Python", "Java"],
|
|
global_index=global_index,
|
|
)
|
|
assert "BM25+RRF" in str(result["Python"]["rationale"])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BM25 + auto-tagging (mocked AutoTagger)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_auto_tagging_expands_candidate_list() -> None:
|
|
"""AutoTagger injects 'Machine Learning' into ['ML'] → score > 0.
|
|
|
|
A global index is provided that includes 'Machine Learning' (simulating
|
|
it existing in another candidate's original skill list in the pool).
|
|
After expansion the working list contains 'Machine Learning', which is
|
|
also in the global corpus, so BM25 can assign it a positive IDF score.
|
|
"""
|
|
mock_tagger = MagicMock()
|
|
mock_tagger.expand_competences = AsyncMock(
|
|
return_value=["ML", "Machine Learning"]
|
|
)
|
|
|
|
engine = _bm25_engine(use_auto_tagging=True, auto_tagger=mock_tagger)
|
|
global_index = Bm25Index(
|
|
corpus=["ML", "Machine Learning", "Python", "JavaScript", "Java"]
|
|
)
|
|
result = await engine.compute_competence_similarity(
|
|
required=["Machine Learning"],
|
|
candidate=["ML"],
|
|
global_index=global_index,
|
|
)
|
|
assert float(cast(Any, result["Machine Learning"]["score"])) > 0.0
|
|
mock_tagger.expand_competences.assert_called_once_with(
|
|
["Machine Learning"], ["ML"]
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_auto_tagging_disabled_uses_raw_candidate() -> None:
|
|
"""When use_auto_tagging=False, AutoTagger is never called."""
|
|
mock_tagger = MagicMock()
|
|
mock_tagger.expand_competences = AsyncMock()
|
|
|
|
engine = _bm25_engine(use_auto_tagging=False, auto_tagger=mock_tagger)
|
|
await engine.compute_competence_similarity(
|
|
required=["Machine Learning"],
|
|
candidate=["ML"],
|
|
)
|
|
mock_tagger.expand_competences.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bm25_global_index_filters_to_candidate_subset() -> None:
|
|
"""Global index is queried and results filtered to the candidate's own skills.
|
|
|
|
A global corpus of seven skills is provided; the candidate only knows
|
|
'Python'. The engine must return a positive score for the 'Python'
|
|
required competence because the global index assigns it positive IDF,
|
|
and all other global corpus items are excluded by the filter.
|
|
"""
|
|
engine = _bm25_engine()
|
|
global_index = Bm25Index(
|
|
corpus=["Python", "JavaScript", "Java", "Go", "Ruby", "Rust", "TypeScript"]
|
|
)
|
|
result = await engine.compute_competence_similarity(
|
|
required=["Python"],
|
|
candidate=["Python"],
|
|
global_index=global_index,
|
|
)
|
|
assert result["Python"]["score"] == pytest.approx(1.0)
|
|
assert result["Python"]["best_match"] == "Python"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LLM-based role similarity
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_role_similarity_bad_roles_return_zero() -> None:
|
|
"""None, empty, and '(unknown)' roles return 0.0 without LLM call."""
|
|
engine = _bm25_engine()
|
|
assert await engine.compute_role_similarity(None, "Developer") == 0.0
|
|
assert await engine.compute_role_similarity("Developer", None) == 0.0
|
|
assert await engine.compute_role_similarity("", "Developer") == 0.0
|
|
assert await engine.compute_role_similarity("Developer", "") == 0.0
|
|
assert await engine.compute_role_similarity(" ", "Developer") == 0.0
|
|
assert await engine.compute_role_similarity("(unknown)", "Developer") == 0.0
|
|
assert await engine.compute_role_similarity("(Unknown)", "Developer") == 0.0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_role_similarity_identical_roles_return_one() -> None:
|
|
"""Identical roles (case-insensitive) return 1.0 without LLM call."""
|
|
engine = _bm25_engine()
|
|
assert await engine.compute_role_similarity("Developer", "Developer") == 1.0
|
|
assert await engine.compute_role_similarity("Developer", "developer") == 1.0
|
|
assert await engine.compute_role_similarity("BACKEND DEV", "backend dev") == 1.0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_role_similarity_calls_llm_and_parses_response() -> None:
|
|
"""LLM returns a similarity score that is parsed and returned."""
|
|
mock_client = AsyncMock()
|
|
mock_client.chat_completion = AsyncMock(
|
|
return_value=json.dumps({"similarity": 0.75})
|
|
)
|
|
|
|
engine = SimilarityEngine(client=mock_client)
|
|
score = await engine.compute_role_similarity("Backend Developer", "Software Engineer")
|
|
assert score == pytest.approx(0.75)
|
|
mock_client.chat_completion.assert_called_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_role_similarity_caches_result_symmetrically() -> None:
|
|
"""Second call with swapped roles uses cache, no second LLM call."""
|
|
mock_client = AsyncMock()
|
|
mock_client.chat_completion = AsyncMock(
|
|
return_value=json.dumps({"similarity": 0.8})
|
|
)
|
|
|
|
engine = SimilarityEngine(client=mock_client)
|
|
score1 = await engine.compute_role_similarity("Role A", "Role B")
|
|
score2 = await engine.compute_role_similarity("Role B", "Role A")
|
|
|
|
assert score1 == 0.8
|
|
assert score2 == 0.8
|
|
# Only one LLM call should have been made
|
|
assert mock_client.chat_completion.call_count == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_role_similarity_clamps_score() -> None:
|
|
"""Scores outside [0, 1] are clamped."""
|
|
mock_client = AsyncMock()
|
|
mock_client.chat_completion = AsyncMock(
|
|
return_value=json.dumps({"similarity": 1.5})
|
|
)
|
|
|
|
engine = SimilarityEngine(client=mock_client)
|
|
score = await engine.compute_role_similarity("A", "B")
|
|
assert score == 1.0
|
|
|
|
# Reset for negative test
|
|
mock_client.chat_completion = AsyncMock(
|
|
return_value=json.dumps({"similarity": -0.3})
|
|
)
|
|
engine2 = SimilarityEngine(client=mock_client)
|
|
score2 = await engine2.compute_role_similarity("C", "D")
|
|
assert score2 == 0.0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_role_similarity_exception_returns_zero() -> None:
|
|
"""On any exception, return 0.0 and do not cache."""
|
|
mock_client = AsyncMock()
|
|
mock_client.chat_completion = AsyncMock(side_effect=RuntimeError("API down"))
|
|
|
|
engine = SimilarityEngine(client=mock_client)
|
|
score = await engine.compute_role_similarity("A", "B")
|
|
assert score == 0.0
|
|
|
|
# Cache should be empty (failed results not cached)
|
|
assert len(engine._role_similarity_cache) == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_role_similarity_invalid_json_returns_zero() -> None:
|
|
"""Invalid JSON from LLM returns 0.0."""
|
|
mock_client = AsyncMock()
|
|
mock_client.chat_completion = AsyncMock(return_value="not json at all")
|
|
|
|
engine = SimilarityEngine(client=mock_client)
|
|
score = await engine.compute_role_similarity("A", "B")
|
|
assert score == 0.0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_role_similarity_missing_key_returns_zero() -> None:
|
|
"""JSON without 'similarity' key returns 0.0."""
|
|
mock_client = AsyncMock()
|
|
mock_client.chat_completion = AsyncMock(
|
|
return_value=json.dumps({"score": 0.9}) # wrong key
|
|
)
|
|
|
|
engine = SimilarityEngine(client=mock_client)
|
|
score = await engine.compute_role_similarity("A", "B")
|
|
assert score == 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# clear_role_cache
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_role_cache() -> None:
|
|
"""clear_role_cache empties the cache."""
|
|
mock_client = AsyncMock()
|
|
mock_client.chat_completion = AsyncMock(
|
|
return_value=json.dumps({"similarity": 0.6})
|
|
)
|
|
|
|
engine = SimilarityEngine(client=mock_client)
|
|
await engine.compute_role_similarity("A", "B")
|
|
assert len(engine._role_similarity_cache) == 1
|
|
|
|
engine.clear_role_cache()
|
|
assert len(engine._role_similarity_cache) == 0
|